(paimon) branch master updated: [core] Reduce get partition requests in migrating hive table (#5364)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new a3f4ab32bd [core] Reduce get partition requests in migrating hive 
table (#5364)
a3f4ab32bd is described below

commit a3f4ab32bd8b2a6988ae537067123f68213ccfea
Author: Zouxxyy <zouxinyu....@alibaba-inc.com>
AuthorDate: Thu Mar 27 23:08:35 2025 +0800

    [core] Reduce get partition requests in migrating hive table (#5364)
---
 .../org/apache/paimon/migrate/FileMetaUtils.java   |  4 ++--
 .../apache/paimon/hive/migrate/HiveMigrator.java   | 28 +++++++---------------
 2 files changed, 11 insertions(+), 21 deletions(-)

diff --git 
a/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java 
b/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java
index e72120abac..6f43ada6c9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java
@@ -240,7 +240,7 @@ public class FileMetaUtils {
 
     public static BinaryRow writePartitionValue(
             RowType partitionRowType,
-            Map<String, String> partitionValues,
+            List<String> partitionValues,
             List<BinaryWriter.ValueSetter> valueSetters,
             String partitionDefaultName) {
 
@@ -250,7 +250,7 @@ public class FileMetaUtils {
         List<DataField> fields = partitionRowType.getFields();
 
         for (int i = 0; i < fields.size(); i++) {
-            String partitionName = partitionValues.get(fields.get(i).name());
+            String partitionName = partitionValues.get(i);
             if (partitionName.equals(partitionDefaultName)) {
                 binaryRowWriter.setNullAt(i);
             } else {
diff --git 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java
 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java
index 74f82cbbbe..6450cb87c2 100644
--- 
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java
+++ 
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java
@@ -163,25 +163,20 @@ public class HiveMigrator implements Migrator {
             FileStoreTable paimonTable = (FileStoreTable) 
hiveCatalog.getTable(identifier);
             checkPaimonTable(paimonTable);
 
-            List<String> partitionsNames =
-                    client.listPartitionNames(sourceDatabase, sourceTable, 
Short.MAX_VALUE);
+            List<Partition> partitions =
+                    client.listPartitions(sourceDatabase, sourceTable, 
Short.MAX_VALUE);
             checkCompatible(sourceHiveTable, paimonTable);
 
             List<MigrateTask> tasks = new ArrayList<>();
             Map<Path, Path> rollBack = new ConcurrentHashMap<>();
-            if (partitionsNames.isEmpty()) {
+            if (partitions.isEmpty()) {
                 tasks.add(
                         importUnPartitionedTableTask(
                                 fileIO, sourceHiveTable, paimonTable, 
rollBack));
             } else {
                 tasks.addAll(
                         importPartitionedTableTask(
-                                client,
-                                fileIO,
-                                partitionsNames,
-                                sourceHiveTable,
-                                paimonTable,
-                                rollBack));
+                                fileIO, partitions, sourceHiveTable, 
paimonTable, rollBack));
             }
 
             List<Future<CommitMessage>> futures =
@@ -292,13 +287,11 @@ public class HiveMigrator implements Migrator {
     }
 
     private List<MigrateTask> importPartitionedTableTask(
-            IMetaStoreClient client,
             FileIO fileIO,
-            List<String> partitionNames,
+            List<Partition> partitions,
             Table sourceTable,
             FileStoreTable paimonTable,
-            Map<Path, Path> rollback)
-            throws Exception {
+            Map<Path, Path> rollback) {
         List<MigrateTask> migrateTasks = new ArrayList<>();
         List<BinaryWriter.ValueSetter> valueSetters = new ArrayList<>();
 
@@ -309,17 +302,14 @@ public class HiveMigrator implements Migrator {
                 .getFieldTypes()
                 .forEach(type -> 
valueSetters.add(BinaryWriter.createValueSetter(type)));
 
-        for (String partitionName : partitionNames) {
-            Partition partition =
-                    client.getPartition(
-                            sourceTable.getDbName(), 
sourceTable.getTableName(), partitionName);
-            Map<String, String> values = 
client.partitionNameToSpec(partitionName);
+        for (Partition partition : partitions) {
+            List<String> partitionValues = partition.getValues();
             String format = 
parseFormat(partition.getSd().getSerdeInfo().toString());
             String location = partition.getSd().getLocation();
             BinaryRow partitionRow =
                     FileMetaUtils.writePartitionValue(
                             partitionRowType,
-                            values,
+                            partitionValues,
                             valueSetters,
                             coreOptions.partitionDefaultName());
             Path path = 
paimonTable.store().pathFactory().bucketPath(partitionRow, 0);