This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2266f92cb9d [fix](hive) Key Hive partition cache by name (#64867)
2266f92cb9d is described below
commit 2266f92cb9d0e3bf32243d1cc0a72157a3ad7f17
Author: Socrates <[email protected]>
AuthorDate: Fri Jun 26 21:49:46 2026 +0800
[fix](hive) Key Hive partition cache by name (#64867)
Problem Summary: Hive external partition pruning and metadata paths used
Doris-local partition ids inside `HivePartitionValues`. The original ids
were generated from partition names, so different partition names could
collide and cause partition item/value lookups to read the wrong entry.
This PR changes the Hive external partition cache to use Hive partition
names as the stable key instead of a local numeric id.
The current implementation stores `Map<String, PartitionItem>` and
`Map<String, List<String>>` in `HivePartitionValues`, rebuilds sorted
partition ranges directly from the partition-name keyed items, and
updates incremental add/drop cache operations by partition name. Callers
that need all partition values now consume the partition-name keyed maps
directly.
---
.../doris/datasource/hive/HMSExternalTable.java | 26 +++----
.../apache/doris/datasource/hive/HiveDlaTable.java | 23 +++---
.../datasource/hive/HiveExternalMetaCache.java | 86 +++++++---------------
.../org/apache/doris/planner/HiveTableSink.java | 2 +-
.../doris/tablefunction/MetadataGenerator.java | 4 +-
.../datasource/hive/HMSExternalTableTest.java | 9 +--
.../datasource/hive/HiveMetaStoreCacheTest.java | 15 ++++
.../doris/planner/ListPartitionPrunerV2Test.java | 12 +--
.../hive/test_hive_partition_values_tvf.out | 3 +-
.../hive/test_hive_partition_values_tvf.groovy | 5 +-
10 files changed, 74 insertions(+), 111 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 1e42c7878c2..1d234377d83 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -77,7 +77,6 @@ import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
-import com.google.common.collect.BiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -497,14 +496,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
HiveExternalMetaCache.HivePartitionValues hivePartitionValues =
getHivePartitionValues(
MvccUtil.getSnapshotFromContext(this));
- Map<Long, PartitionItem> idToPartitionItem =
hivePartitionValues.getIdToPartitionItem();
- // transfer id to name
- BiMap<Long, String> idToName =
hivePartitionValues.getPartitionNameToIdMap().inverse();
- Map<String, PartitionItem> nameToPartitionItem =
Maps.newHashMapWithExpectedSize(idToPartitionItem.size());
- for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
- nameToPartitionItem.put(idToName.get(entry.getKey()),
entry.getValue());
- }
- return nameToPartitionItem;
+ return Maps.newHashMap(hivePartitionValues.getNameToPartitionItem());
}
public boolean isHiveTransactionalTable() {
@@ -1061,7 +1053,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.hive(getCatalog().getId());
List<HivePartition> partitionList =
cache.getAllPartitionsWithCache(this,
-
Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values()));
+
Lists.newArrayList(hivePartitionValues.getNameToPartitionValues().values()));
if (CollectionUtils.isEmpty(partitionList)) {
return 0;
}
@@ -1115,7 +1107,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return UNKNOWN_ROW_COUNT;
}
- int totalPartitionSize = partitionValues == null ? 1 :
partitionValues.getIdToPartitionItem().size();
+ int totalPartitionSize = partitionValues == null ? 1 :
partitionValues.getNameToPartitionItem().size();
if (samplePartitionSize != 0 && samplePartitionSize <
totalPartitionSize) {
LOG.info("Table {} sampled {} of {} partitions, sampled size
is {}",
name, samplePartitionSize, totalPartitionSize,
totalSize);
@@ -1144,11 +1136,11 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
// no need to worry that this call will invalid or refresh the
cache.
// because it has enough space to keep partition info of all
tables in cache.
partitionValues = getHivePartitionValues(snapshot);
- if (partitionValues == null ||
partitionValues.getPartitionNameToIdMap() == null) {
+ if (partitionValues == null ||
partitionValues.getNameToPartitionItem() == null) {
LOG.warn("Partition values for hive table {} is null", name);
} else {
LOG.info("Partition values size for hive table {} is {}",
- name,
partitionValues.getPartitionNameToIdMap().size());
+ name, partitionValues.getNameToPartitionItem().size());
}
}
return partitionValues;
@@ -1165,18 +1157,18 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
.hive(getCatalog().getId());
List<HivePartition> hivePartitions = Lists.newArrayList();
if (partitionValues != null) {
- Map<Long, PartitionItem> idToPartitionItem =
partitionValues.getIdToPartitionItem();
- int totalPartitionSize = idToPartitionItem.size();
+ Map<String, PartitionItem> nameToPartitionItem =
partitionValues.getNameToPartitionItem();
+ int totalPartitionSize = nameToPartitionItem.size();
Collection<PartitionItem> partitionItems;
List<List<String>> partitionValuesList;
// If partition number is too large, randomly choose part of them
to estimate the whole table.
if (sampleSize > 0 && sampleSize < totalPartitionSize) {
- List<PartitionItem> items = new
ArrayList<>(idToPartitionItem.values());
+ List<PartitionItem> items = new
ArrayList<>(nameToPartitionItem.values());
Collections.shuffle(items);
partitionItems = items.subList(0, sampleSize);
partitionValuesList =
Lists.newArrayListWithCapacity(sampleSize);
} else {
- partitionItems = idToPartitionItem.values();
+ partitionItems = nameToPartitionItem.values();
partitionValuesList =
Lists.newArrayListWithCapacity(totalPartitionSize);
}
for (PartitionItem item : partitionItems) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
index bfd39c59684..6b0a4c8e65f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
@@ -72,10 +72,10 @@ public class HiveDlaTable extends HMSDlaTable {
public MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot) throws AnalysisException {
HiveExternalMetaCache.HivePartitionValues hivePartitionValues =
hmsTable.getHivePartitionValues(snapshot);
- Long partitionId =
getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues);
+ checkPartitionExists(partitionName, hivePartitionValues);
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.hive(hmsTable.getCatalog().getId());
- HivePartition hivePartition =
getHivePartitionByIdOrAnalysisException(partitionId,
+ HivePartition hivePartition =
getHivePartitionByNameOrAnalysisException(partitionName,
hivePartitionValues, cache);
return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime());
}
@@ -98,7 +98,7 @@ public class HiveDlaTable extends HMSDlaTable {
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.hive(hmsTable.getCatalog().getId());
List<HivePartition> partitionList =
cache.getAllPartitionsWithCache(hmsTable,
-
Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values()));
+
Lists.newArrayList(hivePartitionValues.getNameToPartitionValues().values()));
if (CollectionUtils.isEmpty(partitionList)) {
return new MTMVMaxTimestampSnapshot(hmsTable.getName(), 0L);
}
@@ -113,26 +113,23 @@ public class HiveDlaTable extends HMSDlaTable {
hmsTable.getPartitionColumns()), maxVersionTime);
}
- private Long getPartitionIdByNameOrAnalysisException(String partitionName,
- HiveExternalMetaCache.HivePartitionValues hivePartitionValues)
- throws AnalysisException {
- Long partitionId =
hivePartitionValues.getPartitionNameToIdMap().get(partitionName);
- if (partitionId == null) {
+ private void checkPartitionExists(String partitionName,
+ HiveExternalMetaCache.HivePartitionValues hivePartitionValues)
throws AnalysisException {
+ if
(!hivePartitionValues.getNameToPartitionValues().containsKey(partitionName)) {
throw new AnalysisException("can not find partition: " +
partitionName);
}
- return partitionId;
}
- private HivePartition getHivePartitionByIdOrAnalysisException(Long
partitionId,
+ private HivePartition getHivePartitionByNameOrAnalysisException(String
partitionName,
HiveExternalMetaCache.HivePartitionValues hivePartitionValues,
HiveExternalMetaCache cache) throws AnalysisException {
- List<String> partitionValues =
hivePartitionValues.getPartitionValuesMap().get(partitionId);
+ List<String> partitionValues =
hivePartitionValues.getNameToPartitionValues().get(partitionName);
if (CollectionUtils.isEmpty(partitionValues)) {
- throw new AnalysisException("can not find partitionValues: " +
partitionId);
+ throw new AnalysisException("can not find partitionValues: " +
partitionName);
}
HivePartition partition = cache.getHivePartition(hmsTable,
partitionValues);
if (partition == null) {
- throw new AnalysisException("can not find partition: " +
partitionId);
+ throw new AnalysisException("can not find partition: " +
partitionName);
}
return partition;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
index efcde181758..abc68febcba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
@@ -52,13 +52,10 @@ import org.apache.doris.fs.DirectoryLister;
import org.apache.doris.fs.FileSystemCache;
import org.apache.doris.fs.FileSystemDirectoryLister;
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
-import org.apache.doris.planner.ListPartitionPrunerV2;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Streams;
@@ -289,19 +286,15 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
LOG.debug("load #{} partitions for {} in catalog {}",
partitionNames.size(), key, catalog.getName());
}
- Map<Long, PartitionItem> idToPartitionItem =
Maps.newHashMapWithExpectedSize(partitionNames.size());
- BiMap<String, Long> partitionNameToIdMap =
HashBiMap.create(partitionNames.size());
- String localDbName = nameMapping.getLocalDbName();
- String localTblName = nameMapping.getLocalTblName();
+ Map<String, PartitionItem> nameToPartitionItem =
Maps.newHashMapWithExpectedSize(partitionNames.size());
+ Map<String, List<String>> nameToPartitionValues =
Maps.newHashMapWithExpectedSize(partitionNames.size());
for (String partitionName : partitionNames) {
- long partitionId = Util.genIdByName(catalog.getName(),
localDbName, localTblName, partitionName);
ListPartitionItem listPartitionItem =
toListPartitionItem(partitionName, key.types, catalog.getName());
- idToPartitionItem.put(partitionId, listPartitionItem);
- partitionNameToIdMap.put(partitionName, partitionId);
+ nameToPartitionItem.put(partitionName, listPartitionItem);
+ nameToPartitionValues.put(partitionName,
HiveUtil.toPartitionValues(partitionName));
}
- Map<Long, List<String>> partitionValuesMap =
ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem);
- return new HivePartitionValues(idToPartitionItem,
partitionNameToIdMap, partitionValuesMap);
+ return new HivePartitionValues(nameToPartitionItem,
nameToPartitionValues);
}
private ListPartitionItem toListPartitionItem(String partitionName,
List<Type> types, String catalogName) {
@@ -640,12 +633,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
return;
}
- Long partitionId =
partitionValues.partitionNameToIdMap.get(partitionName);
- if (partitionId == null) {
- return;
- }
-
- List<String> values =
partitionValues.partitionValuesMap.get(partitionId);
+ List<String> values =
partitionValues.nameToPartitionValues.get(partitionName);
if (values == null) {
return;
}
@@ -728,29 +716,22 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
}
HivePartitionValues copy = partitionValues.copy();
- Map<Long, PartitionItem> idToPartitionItemBefore =
copy.getIdToPartitionItem();
- Map<String, Long> partitionNameToIdMapBefore =
copy.getPartitionNameToIdMap();
- Map<Long, PartitionItem> idToPartitionItem = new HashMap<>();
+ Map<String, PartitionItem> nameToPartitionItem =
copy.getNameToPartitionItem();
+ Map<String, List<String>> nameToPartitionValues =
copy.getNameToPartitionValues();
HMSExternalCatalog catalog = hmsCatalog(catalogId);
- String localDbName = nameMapping.getLocalDbName();
String localTblName = nameMapping.getLocalTblName();
for (String partitionName : partitionNames) {
- if (partitionNameToIdMapBefore.containsKey(partitionName)) {
+ if (nameToPartitionItem.containsKey(partitionName)) {
LOG.info("addPartitionsCache partitionName:[{}] has exist
in table:[{}]",
partitionName, localTblName);
continue;
}
- long partitionId = Util.genIdByName(catalog.getName(),
localDbName, localTblName, partitionName);
ListPartitionItem listPartitionItem =
toListPartitionItem(partitionName, key.types, catalog.getName());
- idToPartitionItemBefore.put(partitionId, listPartitionItem);
- idToPartitionItem.put(partitionId, listPartitionItem);
- partitionNameToIdMapBefore.put(partitionName, partitionId);
+ nameToPartitionItem.put(partitionName, listPartitionItem);
+ nameToPartitionValues.put(partitionName,
HiveUtil.toPartitionValues(partitionName));
}
- Map<Long, List<String>> partitionValuesMapBefore =
copy.getPartitionValuesMap();
- Map<Long, List<String>> partitionValuesMap =
ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem);
- partitionValuesMapBefore.putAll(partitionValuesMap);
copy.rebuildSortedPartitionRanges();
HivePartitionValues partitionValuesCur =
partitionValuesEntry.getIfPresent(key);
@@ -778,19 +759,17 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
}
HivePartitionValues copy = partitionValues.copy();
- Map<String, Long> partitionNameToIdMapBefore =
copy.getPartitionNameToIdMap();
- Map<Long, PartitionItem> idToPartitionItemBefore =
copy.getIdToPartitionItem();
- Map<Long, List<String>> partitionValuesMap =
copy.getPartitionValuesMap();
+ Map<String, PartitionItem> nameToPartitionItem =
copy.getNameToPartitionItem();
+ Map<String, List<String>> nameToPartitionValues =
copy.getNameToPartitionValues();
for (String partitionName : partitionNames) {
- if (!partitionNameToIdMapBefore.containsKey(partitionName)) {
+ if (!nameToPartitionItem.containsKey(partitionName)) {
LOG.info("dropPartitionsCache partitionName:[{}] not exist
in table:[{}]",
partitionName, nameMapping.getFullLocalName());
continue;
}
- Long partitionId =
partitionNameToIdMapBefore.remove(partitionName);
- idToPartitionItemBefore.remove(partitionId);
- partitionValuesMap.remove(partitionId);
+ nameToPartitionItem.remove(partitionName);
+ nameToPartitionValues.remove(partitionName);
if (invalidPartitionCache) {
invalidatePartitionCache(dorisTable, partitionName);
@@ -1060,9 +1039,8 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
@Data
public static class HivePartitionValues {
- private BiMap<String, Long> partitionNameToIdMap;
- private Map<Long, PartitionItem> idToPartitionItem;
- private Map<Long, List<String>> partitionValuesMap;
+ private Map<String, PartitionItem> nameToPartitionItem;
+ private Map<String, List<String>> nameToPartitionValues;
// Sorted partition ranges for binary search filtering.
private SortedPartitionRanges<String> sortedPartitionRanges;
@@ -1070,20 +1048,18 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
public HivePartitionValues() {
}
- public HivePartitionValues(Map<Long, PartitionItem> idToPartitionItem,
- BiMap<String, Long> partitionNameToIdMap,
- Map<Long, List<String>> partitionValuesMap) {
- this.idToPartitionItem = idToPartitionItem;
- this.partitionNameToIdMap = partitionNameToIdMap;
- this.partitionValuesMap = partitionValuesMap;
+ public HivePartitionValues(Map<String, PartitionItem>
nameToPartitionItem,
+ Map<String, List<String>> nameToPartitionValues) {
+ this.nameToPartitionItem = nameToPartitionItem;
+ this.nameToPartitionValues = nameToPartitionValues;
this.sortedPartitionRanges = buildSortedPartitionRanges();
}
public HivePartitionValues copy() {
HivePartitionValues copy = new HivePartitionValues();
- copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null :
HashBiMap.create(partitionNameToIdMap));
- copy.setIdToPartitionItem(idToPartitionItem == null ? null :
Maps.newHashMap(idToPartitionItem));
- copy.setPartitionValuesMap(partitionValuesMap == null ? null :
Maps.newHashMap(partitionValuesMap));
+ copy.setNameToPartitionItem(nameToPartitionItem == null ? null :
Maps.newHashMap(nameToPartitionItem));
+ copy.setNameToPartitionValues(
+ nameToPartitionValues == null ? null :
Maps.newHashMap(nameToPartitionValues));
return copy;
}
@@ -1096,20 +1072,10 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
}
private SortedPartitionRanges<String> buildSortedPartitionRanges() {
- if (partitionNameToIdMap == null || partitionNameToIdMap.isEmpty()
- || idToPartitionItem == null ||
idToPartitionItem.isEmpty()) {
+ if (nameToPartitionItem == null || nameToPartitionItem.isEmpty()) {
return null;
}
- BiMap<Long, String> idToName = partitionNameToIdMap.inverse();
- Map<String, PartitionItem> nameToPartitionItem =
Maps.newHashMapWithExpectedSize(idToPartitionItem.size());
- for (Map.Entry<Long, PartitionItem> entry :
idToPartitionItem.entrySet()) {
- String partitionName = idToName.get(entry.getKey());
- if (partitionName != null) {
- nameToPartitionItem.put(partitionName, entry.getValue());
- }
- }
-
return SortedPartitionRanges.build(nameToPartitionItem);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
index a9568e225f0..dd7dc1b120a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
@@ -213,7 +213,7 @@ public class HiveTableSink extends
BaseExternalTableDataSink {
HiveExternalMetaCache.HivePartitionValues partitionValues =
targetTable.getHivePartitionValues(MvccUtil.getSnapshotFromContext(targetTable));
List<List<String>> partitionValuesList =
- new
ArrayList<>(partitionValues.getPartitionValuesMap().values());
+ new
ArrayList<>(partitionValues.getNameToPartitionValues().values());
hivePartitions = cache.getAllPartitionsWithCache(targetTable,
partitionValuesList);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index f4d9d535f84..1838085cb1f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -2116,9 +2116,9 @@ public class MetadataGenerator {
HiveExternalMetaCache.HivePartitionValues hivePartitionValues =
tbl.getHivePartitionValues(
MvccUtil.getSnapshotFromContext(tbl));
- Map<Long, List<String>> valuesMap =
hivePartitionValues.getPartitionValuesMap();
+ Map<String, List<String>> valuesMap =
hivePartitionValues.getNameToPartitionValues();
List<TRow> dataBatch = Lists.newArrayList();
- for (Map.Entry<Long, List<String>> entry : valuesMap.entrySet()) {
+ for (Map.Entry<String, List<String>> entry : valuesMap.entrySet()) {
TRow trow = new TRow();
List<String> values = entry.getValue();
if (values.size() != partitionCols.size()) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
index 2cefa4821da..5cf4779f73e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalTableTest.java
@@ -29,7 +29,6 @@ import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.fs.FileSystemDirectoryLister;
import org.apache.doris.thrift.TFileFormatType;
-import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -43,7 +42,6 @@ import org.mockito.Mockito;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
/**
@@ -299,12 +297,9 @@ public class HMSExternalTableTest {
Lists.newArrayList(Type.VARCHAR),
true);
PartitionItem partitionItem = new
ListPartitionItem(Lists.newArrayList(partitionKey));
- long partitionId = 1L;
- Map<Long, PartitionItem> idToPartitionItem =
ImmutableMap.of(partitionId, partitionItem);
this.partitionValues = new
HiveExternalMetaCache.HivePartitionValues(
- idToPartitionItem,
- HashBiMap.create(ImmutableMap.of("dt=" + partitionValue,
partitionId)),
- ImmutableMap.of(partitionId,
Collections.singletonList(partitionValue)));
+ ImmutableMap.of("dt=" + partitionValue, partitionItem),
+ ImmutableMap.of("dt=" + partitionValue,
Collections.singletonList(partitionValue)));
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
index eb4b28e9a88..931c7ac7e5a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.util.Util;
@@ -30,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicLong;
@@ -123,6 +125,19 @@ public class HiveMetaStoreCacheTest {
}
}
+ @Test
+ public void testHivePartitionValuesCopyKeepsIndependentNameMaps() {
+ Map<String, PartitionItem> nameToPartitionItem = new HashMap<>();
+ Map<String, List<String>> nameToPartitionValues = new HashMap<>();
+ nameToPartitionValues.put("dt=2026-06-26",
Collections.singletonList("2026-06-26"));
+ HiveExternalMetaCache.HivePartitionValues partitionValues =
+ new
HiveExternalMetaCache.HivePartitionValues(nameToPartitionItem,
nameToPartitionValues);
+
+ HiveExternalMetaCache.HivePartitionValues copy =
partitionValues.copy();
+ copy.getNameToPartitionValues().put("dt=2026-06-27",
Collections.singletonList("2026-06-27"));
+
Assertions.assertFalse(partitionValues.getNameToPartitionValues().containsKey("dt=2026-06-27"));
+ }
+
private void putCache(
MetaCacheEntry<HiveExternalMetaCache.FileCacheKey,
HiveExternalMetaCache.FileCacheValue> fileCache,
MetaCacheEntry<HiveExternalMetaCache.PartitionCacheKey,
HivePartition> partitionCache,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java
index c423fcd0315..e5c5c3497c0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/ListPartitionPrunerV2Test.java
@@ -104,8 +104,8 @@ public class ListPartitionPrunerV2Test {
NameMapping nameMapping = NameMapping.createForTest(catalogId,
dbName, tblName);
PartitionValueCacheKey key = new
PartitionValueCacheKey(nameMapping, types);
HiveExternalMetaCache.HivePartitionValues partitionValues =
cache.getPartitionValues(key);
- Assert.assertEquals(1,
partitionValues.getIdToPartitionItem().size());
- List<PartitionKey> items =
partitionValues.getIdToPartitionItem().values().iterator().next().getItems();
+ Assert.assertEquals(1,
partitionValues.getNameToPartitionItem().size());
+ List<PartitionKey> items =
partitionValues.getNameToPartitionItem().values().iterator().next().getItems();
Assert.assertEquals(1, items.size());
PartitionKey partitionKey = items.get(0);
Assert.assertEquals("1.234",
partitionKey.getKeys().get(0).toString());
@@ -117,9 +117,9 @@ public class ListPartitionPrunerV2Test {
cache.addPartitionsCache(nameMapping, values, types);
HiveExternalMetaCache.HivePartitionValues partitionValues2 =
cache.getPartitionValues(
new PartitionValueCacheKey(nameMapping, types));
- Assert.assertEquals(2,
partitionValues2.getIdToPartitionItem().size());
+ Assert.assertEquals(2,
partitionValues2.getNameToPartitionItem().size());
PartitionKey partitionKey2 = null;
- for (PartitionItem partitionItem :
partitionValues2.getIdToPartitionItem().values()) {
+ for (PartitionItem partitionItem :
partitionValues2.getNameToPartitionItem().values()) {
List<PartitionKey> partitionKeys = partitionItem.getItems();
Assert.assertEquals(1, partitionKeys.size());
if
("5.678000".equals(partitionKeys.get(0).getOriginHiveKeys().get(0))) {
@@ -136,8 +136,8 @@ public class ListPartitionPrunerV2Test {
cache.invalidateTableCache(nameMapping);
HiveExternalMetaCache.HivePartitionValues partitionValues3 =
cache.getPartitionValues(
new PartitionValueCacheKey(nameMapping, types));
- Assert.assertEquals(1,
partitionValues3.getIdToPartitionItem().size());
- List<PartitionKey> items3 =
partitionValues3.getIdToPartitionItem().values().iterator().next().getItems();
+ Assert.assertEquals(1,
partitionValues3.getNameToPartitionItem().size());
+ List<PartitionKey> items3 =
partitionValues3.getNameToPartitionItem().values().iterator().next().getItems();
Assert.assertEquals(1, items3.size());
PartitionKey partitionKey3 = items3.get(0);
Assert.assertEquals("1.234",
partitionKey3.getKeys().get(0).toString());
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
index bc69c716277..738854a43d1 100644
---
a/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
+++
b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
@@ -60,9 +60,9 @@ test3
-- !sql61 --
-- !sql71 --
+\N 0.1 test1
\N 0.2 test2
100 0.3 test3
-\N 0.1 test1
-- !sql72 --
@@ -117,4 +117,3 @@ p8 varchar(65533) Yes true \N NONE
\N \N \N \N \N \N \N \N
false -128 -32768 -2147483648 -9223372036854775808 1900-01-01
1899-01-01T23:59:59 \N
false 127 32767 2147483647 9223372036854775807 9999-12-31
0001-01-01T00:00:01.321 boston
-
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
index c1ce220687c..bd50167725f 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
@@ -69,8 +69,8 @@ suite("test_hive_partition_values_tvf", "p0,external") {
sql """drop database if exists internal.partition_values_db"""
sql """create database if not exists internal.partition_values_db"""
sql """create view internal.partition_values_db.v1 as select * from
${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions"""
- qt_sql71 """select * from internal.partition_values_db.v1"""
- qt_sql72 """select t_string, t_int from
internal.partition_values_db.v1 where t_int != "__HIVE_DEFAULT_PARTITION__""""
+ qt_sql71 """select * from internal.partition_values_db.v1 order by
t_int, t_float, t_string"""
+ qt_sql72 """select t_string, t_int from
internal.partition_values_db.v1 where t_int != "__HIVE_DEFAULT_PARTITION__"
order by t_int, t_string"""
qt_sql73 """with v1 as (select t_string, t_int from
internal.partition_values_db.v1 order by t_int, t_float, t_string) select c1
from (select max(t_string) as c1 from v1) x;"""
// 9. test join
@@ -139,4 +139,3 @@ suite("test_hive_partition_values_tvf", "p0,external") {
qt_sql113 """select * from partition_values_all_types\$partitions
order by p1,p2,p3;"""
}
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
