This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 8f6f38606bb branch-4.2: [fix](fe) Scope Hive file cache entries by
table (#68577)
8f6f38606bb is described below
commit 8f6f38606bb60a3c4f0341bce18e1d7da3f7a079
Author: 924060929 <[email protected]>
AuthorDate: Tue Sep 29 21:34:42 2026 +0800
branch-4.2: [fix](fe) Scope Hive file cache entries by table (#68577)
### What problem does this PR solve?
Issue Number: None
Related PR: #68575 (the same branch-4.1 fix)
Problem Summary: Two Hive partition tables can share a partition
location. On branch-4.2, the partition file-list cache key uses the
catalog, location, and partition values, but omits the table ID.
`REFRESH TABLE` invalidates file-list entries by table ID. If table a
loads the old listing, table b can hit it after writing a new file and
refreshing b, so the scan omits the new file and returns incomplete
results. Include the table ID in the cache key's equality and hash code
so each table loads and invalidates its own listing.
The FE unit test uses the same catalog, location, format, and partition
value for a and b. It verifies that b cannot reuse a's cached value,
both entries coexist, and refreshing b removes only b's entry.
### Release note
Fix missing rows after refreshing a Hive table that shares a partition
location with another table.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test: `HiveMetaStoreCacheTest` (13 tests passed on
branch-4.2)
- [ ] Manual test
- [ ] No need to test or manual test
- FE Checkstyle: 0 violations.
- Local full FE build was not completed because this isolated worktree
lacked the complete thirdparty installation; `run buildall` is requested
on this PR.
- Behavior changed:
- [ ] No.
- [x] Yes. `REFRESH TABLE` reloads the current Hive table's file listing
even when another table shares its partition location.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/hive/HiveExternalMetaCache.java | 4 ++-
.../datasource/hive/HiveMetaStoreCacheTest.java | 39 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
index cdf58a9adc6..ce288bc37ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
@@ -1112,6 +1112,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
private String inputFormat;
// The values of partitions.
protected List<String> partitionValues;
+ // Keep cache identity aligned with table-scoped invalidation.
private long id;
public FileCacheKey(long catalogId, long id, String location, String
inputFormat,
@@ -1142,6 +1143,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
return dummyKey == ((FileCacheKey) obj).dummyKey;
}
return catalogId == ((FileCacheKey) obj).catalogId
+ && id == ((FileCacheKey) obj).id
&& location.equals(((FileCacheKey) obj).location)
&& Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
}
@@ -1155,7 +1157,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
if (dummyKey != 0) {
return Objects.hash(dummyKey);
}
- return Objects.hash(catalogId, location, partitionValues);
+ return Objects.hash(catalogId, id, location, partitionValues);
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
index 9cc96f58b7d..f067c879705 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
@@ -104,6 +104,45 @@ public class HiveMetaStoreCacheTest {
}
}
+ @Test
+ public void testRefreshTableWithSharedPartitionLocation() {
+ ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "refresh", 1, false);
+ ThreadPoolExecutor listExecutor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "file", 1, false);
+ try {
+ HiveExternalMetaCache cache = new HiveExternalMetaCache(executor,
listExecutor);
+ cache.initCatalog(0, new HashMap<>());
+ MetaCacheEntry<HiveExternalMetaCache.FileCacheKey,
HiveExternalMetaCache.FileCacheValue> fileCache =
+ cache.entry(0, HiveExternalMetaCache.ENTRY_FILE,
+ HiveExternalMetaCache.FileCacheKey.class,
+ HiveExternalMetaCache.FileCacheValue.class);
+
+ String location = "/warehouse/shared/p=p1";
+ List<String> values = Collections.singletonList("p1");
+ HiveExternalMetaCache.FileCacheKey keyA = new
HiveExternalMetaCache.FileCacheKey(
+ 0, Util.genIdByName("db", "a"), location, "parquet",
values);
+ HiveExternalMetaCache.FileCacheKey keyB = new
HiveExternalMetaCache.FileCacheKey(
+ 0, Util.genIdByName("db", "b"), location, "parquet",
values);
+ HiveExternalMetaCache.FileCacheValue oldFiles = new
HiveExternalMetaCache.FileCacheValue();
+ fileCache.put(keyA, oldFiles);
+
+ // b must list the directory after its write instead of reusing
a's old file list.
+ Assertions.assertNull(fileCache.getIfPresent(keyB));
+ HiveExternalMetaCache.FileCacheValue newFiles = new
HiveExternalMetaCache.FileCacheValue();
+ fileCache.put(keyB, newFiles);
+ Assertions.assertSame(oldFiles, fileCache.getIfPresent(keyA));
+ Assertions.assertSame(newFiles, fileCache.getIfPresent(keyB));
+
+ cache.invalidateTable(0, "db", "b");
+ Assertions.assertNull(fileCache.getIfPresent(keyB));
+ Assertions.assertSame(oldFiles, fileCache.getIfPresent(keyA));
+ } finally {
+ executor.shutdownNow();
+ listExecutor.shutdownNow();
+ }
+ }
+
@Test
public void testInvalidateTableCache() {
ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
