This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7d572db50e [hive] Use partition key order for HMS values when adding
or dropping partitions (#10401)
7d572db50e is described below
commit 7d572db50e1e9bb96a85e09c243c66354408c733
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 9 23:01:56 2026 +0900
[hive] Use partition key order for HMS values when adding or dropping
partitions (#10401)
---
.../java/org/apache/paimon/hive/HiveCatalog.java | 27 +++++++++-
.../org/apache/paimon/hive/HiveCatalogTest.java | 58 ++++++++++++++++++++++
2 files changed, 83 insertions(+), 2 deletions(-)
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 8420010267..c6bfee715e 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -441,8 +441,13 @@ public class HiveCatalog extends AbstractCatalog {
int currentTime = (int) (System.currentTimeMillis() / 1000);
StorageDescriptor sd = hmsTable.getSd();
String dataFilePath = getDataFilePath(tableIdentifier, hmsTable);
+ List<String> partitionKeys =
+ hmsTable.getPartitionKeys().stream()
+ .map(FieldSchema::getName)
+ .collect(Collectors.toList());
List<Partition> hivePartitions = new ArrayList<>();
- for (Map<String, String> partitionSpec : partitions) {
+ for (Map<String, String> spec : partitions) {
+ Map<String, String> partitionSpec =
orderByPartitionKeys(partitionKeys, spec);
Partition hivePartition = new Partition();
StorageDescriptor newSd = new StorageDescriptor(sd);
hivePartition.setDbName(tableIdentifier.getDatabaseName());
@@ -475,7 +480,9 @@ public class HiveCatalog extends AbstractCatalog {
? partitions
:
removePartitionsExistsInOtherBranches(identifier, partitions);
for (Map<String, String> part : metaPartitions) {
- List<String> partitionValues = new ArrayList<>(part.values());
+ List<String> partitionValues =
+ new ArrayList<>(
+ orderByPartitionKeys(schema.partitionKeys(),
part).values());
try {
clients()
.execute(
@@ -514,6 +521,22 @@ public class HiveCatalog extends AbstractCatalog {
new LinkedHashMap<>(partitionSpec), onlyValue);
}
+ /**
+ * Hive metastore matches partition values to partition keys by position,
so reorder the spec by
+ * the partition keys. The spec is returned unchanged if its keys differ
from them.
+ */
+ private static Map<String, String> orderByPartitionKeys(
+ List<String> partitionKeys, Map<String, String> spec) {
+ if (partitionKeys.size() != spec.size() ||
!spec.keySet().containsAll(partitionKeys)) {
+ return spec;
+ }
+ Map<String, String> ordered = new LinkedHashMap<>();
+ for (String key : partitionKeys) {
+ ordered.put(key, spec.get(key));
+ }
+ return ordered;
+ }
+
@Override
public void alterPartitions(Identifier identifier,
List<PartitionStatistics> partitions)
throws TableNotExistException {
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index 6cfd1d4fa9..196baafa88 100644
---
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -62,6 +62,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -729,6 +730,63 @@ public class HiveCatalogTest extends CatalogTestBase {
assertPhysicalDts(identifier);
}
+ @Test
+ public void testCreateAndDropPartitionsWithSpecNotInPartitionKeyOrder()
throws Exception {
+ String databaseName = "test_partition_spec_order";
+ String tableName = "multi_key_table";
+ catalog.dropDatabase(databaseName, true, true);
+ catalog.createDatabase(databaseName, true);
+ Identifier identifier = Identifier.create(databaseName, tableName);
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .column("col", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .column("hh", DataTypes.STRING())
+ .partitionKeys("dt", "hh")
+ .build(),
+ false);
+ IMetaStoreClient hmsClient = ((HiveCatalog) catalog).getHmsClient();
+
+ // register dt=d1/hh=h1 and its mirror dt=h1/hh=d1 in partition key
order
+ catalog.createPartitions(
+ identifier,
+ Arrays.asList(
+ orderedSpec("dt", "d1", "hh", "h1"), orderedSpec("dt",
"h1", "hh", "d1")));
+ assertThat(hmsClient.listPartitions(databaseName, tableName,
Short.MAX_VALUE))
+ .extracting(p -> p.getValues())
+ .containsExactlyInAnyOrder(Arrays.asList("d1", "h1"),
Arrays.asList("h1", "d1"));
+
+ // drop dt=d1/hh=h1 with a spec in reversed key order, the mirror must
be kept
+ catalog.dropPartitions(
+ identifier, Collections.singletonList(orderedSpec("hh", "h1",
"dt", "d1")));
+ assertThat(hmsClient.listPartitions(databaseName, tableName,
Short.MAX_VALUE))
+ .extracting(p -> p.getValues())
+ .containsExactly(Arrays.asList("h1", "d1"));
+
+ // create dt=d9/hh=h9 with a spec in reversed key order
+ catalog.createPartitions(
+ identifier, Collections.singletonList(orderedSpec("hh", "h9",
"dt", "d9")));
+ assertThat(hmsClient.listPartitions(databaseName, tableName,
Short.MAX_VALUE))
+ .extracting(p -> p.getValues())
+ .containsExactlyInAnyOrder(Arrays.asList("h1", "d1"),
Arrays.asList("d9", "h9"));
+ assertThat(
+ hmsClient
+ .getPartition(databaseName, tableName,
Arrays.asList("d9", "h9"))
+ .getSd()
+ .getLocation())
+ .endsWith("/dt=d9/hh=h9");
+ }
+
+ private static Map<String, String> orderedSpec(
+ String key1, String value1, String key2, String value2) {
+ Map<String, String> spec = new LinkedHashMap<>();
+ spec.put(key1, value1);
+ spec.put(key2, value2);
+ return spec;
+ }
+
private void writePartitionRow(Identifier identifier, String dt, String
pk) throws Exception {
BatchWriteBuilder writeBuilder =
catalog.getTable(identifier).newBatchWriteBuilder();
try (BatchTableWrite write = writeBuilder.newWrite();
