This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new b47ebe8dbb [core] hotfix: fix extract partition spec from path when
path only partition value and partition equal filter is empty (#6341)
b47ebe8dbb is described below
commit b47ebe8dbb8ae08fe81fb8601248eed8078dfd60
Author: jerry <[email protected]>
AuthorDate: Fri Sep 26 21:48:17 2025 +0800
[core] hotfix: fix extract partition spec from path when path only
partition value and partition equal filter is empty (#6341)
---
.../apache/paimon/utils/PartitionPathUtils.java | 7 ++-
.../org/apache/paimon/catalog/CatalogTestBase.java | 8 +--
.../paimon/table/format/FormatTableScanTest.java | 57 +++++++++++++++++++++-
3 files changed, 63 insertions(+), 9 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
index 185479d8a5..1aade5fbda 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
@@ -252,11 +252,10 @@ public class PartitionPathUtils {
}
public static LinkedHashMap<String, String>
extractPartitionSpecFromPathOnlyValue(
- Path currPath, List<String> partitionKeys, int partitionNumber) {
+ Path currPath, List<String> partitionKeys) {
LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>();
String[] split = currPath.toString().split(Path.SEPARATOR);
- int equalityPartitionSize = partitionKeys.size() - partitionNumber;
- for (int i = 0; i < equalityPartitionSize; i++) {
+ for (int i = 0; i < partitionKeys.size(); i++) {
fullPartSpec.put(partitionKeys.get(i), split[split.length -
partitionKeys.size() + i]);
}
return fullPartSpec;
@@ -286,7 +285,7 @@ public class PartitionPathUtils {
ret.add(
Pair.of(
extractPartitionSpecFromPathOnlyValue(
- part.getPath(), partitionKeys,
partitionNumber),
+ part.getPath(), partitionKeys),
part.getPath()));
} else {
ret.add(Pair.of(extractPartitionSpecFromPath(part.getPath()),
part.getPath()));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
index 05da64ed49..55d03ec6b0 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
@@ -641,11 +641,13 @@ public abstract class CatalogTestBase {
compressionType.value(),
null);
write(factory, dataFilePathFactory.newPath(),
compressionType.value(), datas);
+ List<InternalRow> readAllData = read(table, null, null, null,
null);
+ assertThat(readAllData).containsExactlyInAnyOrder(datas);
Map<String, String> partitionSpec = new HashMap<>();
- partitionSpec.put("dt", "" + dtPartitionValue);
- partitionSpec.put("dt2", dt2PartitionValue);
+ partitionSpec.put("dt", "" + dtPartitionValue + 1);
+ partitionSpec.put("dt2", dt2PartitionValue + 1);
List<InternalRow> readFilterData = read(table, null, null,
partitionSpec, null);
- assertThat(readFilterData).containsExactlyInAnyOrder(datas);
+ assertThat(readFilterData).isEmpty();
catalog.dropTable(Identifier.create(dbName, format), true);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
index 87678e1b57..5bf13d9209 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableScanTest.java
@@ -146,6 +146,44 @@ public class FormatTableScanTest {
assertThat(result.getRight()).isEqualTo(2);
}
+ @TestTemplate
+ void testComputeScanPathWithoutFilter() throws IOException {
+ Path tableLocation = new Path(tmpPath.toUri());
+ PredicateBuilder builder = new PredicateBuilder(partitionType);
+ Predicate predicate = PredicateBuilder.and(builder.greaterThan(0,
2022));
+ PartitionPredicate partitionFilter =
+ PartitionPredicate.fromPredicate(partitionType, predicate);
+ Pair<Path, Integer> result =
+ FormatTableScan.computeScanPathAndLevel(
+ tableLocation,
+ partitionKeys,
+ partitionFilter,
+ partitionType,
+ enablePartitionValueOnly);
+
+ // Should optimize to specific partition path for first key
+ assertThat(result.getLeft()).isEqualTo(tableLocation);
+ assertThat(result.getRight()).isEqualTo(2);
+
+ // test searchPartSpecAndPaths
+ LocalFileIO fileIO = LocalFileIO.create();
+ String partitionPath = enablePartitionValueOnly ? "2023/12" :
"year=2023/month=12";
+ fileIO.mkdirs(new Path(tableLocation, partitionPath));
+ List<Pair<LinkedHashMap<String, String>, Path>> searched =
+ searchPartSpecAndPaths(
+ fileIO,
+ result.getLeft(),
+ result.getRight(),
+ partitionKeys,
+ enablePartitionValueOnly);
+ LinkedHashMap<String, String> expectPartitionSpec =
+ new LinkedHashMap<>(partitionKeys.size());
+ expectPartitionSpec.put("year", "2023");
+ expectPartitionSpec.put("month", "12");
+ assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
+ assertThat(searched.size()).isEqualTo(1);
+ }
+
@TestTemplate
void testGetScanPathAndLevelWithEqualityFilter() throws IOException {
Path tableLocation = new Path(tmpPath.toUri());
@@ -176,8 +214,13 @@ public class FormatTableScanTest {
fileIO,
result.getLeft(),
result.getRight(),
- null,
+ partitionKeys,
enablePartitionValueOnly);
+ LinkedHashMap<String, String> expectPartitionSpec =
+ new LinkedHashMap<>(partitionKeys.size());
+ expectPartitionSpec.put("year", "2023");
+ expectPartitionSpec.put("month", "12");
+ assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
assertThat(searched.size()).isEqualTo(1);
}
@@ -208,7 +251,17 @@ public class FormatTableScanTest {
partitionPath = enablePartitionValueOnly ? "2023/12" :
"year=2023/month=12";
fileIO.mkdirs(new Path(tableLocation, partitionPath));
List<Pair<LinkedHashMap<String, String>, Path>> searched =
- searchPartSpecAndPaths(fileIO, result.getLeft(),
result.getRight(), null, false);
+ searchPartSpecAndPaths(
+ fileIO,
+ result.getLeft(),
+ result.getRight(),
+ partitionKeys,
+ enablePartitionValueOnly);
+ LinkedHashMap<String, String> expectPartitionSpec =
+ new LinkedHashMap<>(partitionKeys.size());
+ expectPartitionSpec.put("year", "2023");
+ expectPartitionSpec.put("month", "12");
+ assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
assertThat(searched.size()).isEqualTo(1);
}
