This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 1217eaaf11d9 fix(flink): avoid partition stats pruning for incremental 
reads (#20481)
1217eaaf11d9 is described below

commit 1217eaaf11d9ac177f718e3619e1df86c8403148
Author: Shuo Cheng <[email protected]>
AuthorDate: Wed Nov 4 10:19:03 2026 +0800

    fix(flink): avoid partition stats pruning for incremental reads (#20481)
---
 .../org/apache/hudi/table/HoodieTableSource.java   |  5 ++-
 .../hudi/source/TestIncrementalInputSplits.java    | 10 ++++-
 .../apache/hudi/table/ITTestHoodieDataSource.java  | 47 +++++++++++++++++++++-
 .../apache/hudi/table/TestHoodieTableSource.java   | 27 +++++++++++++
 4 files changed, 86 insertions(+), 3 deletions(-)

diff --git 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
index a32014a3c67f..0e7d54ec35a6 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
@@ -399,7 +399,10 @@ public class HoodieTableSource extends FileIndexReader 
implements
     Tuple2<List<ResolvedExpression>, List<ResolvedExpression>> splitFilters = 
splitExprByPartitionCall(simpleFilters, this.partitionKeys, this.tableRowType);
     this.predicates = ExpressionPredicates.fromExpression(splitFilters.f0);
     this.columnStatsProbe = ColumnStatsProbe.newInstance(splitFilters.f0);
-    this.partitionPruner = createPartitionPruner(splitFilters.f1, 
columnStatsProbe);
+    // Partition stats only reflect the latest table state and can exclude 
values in historical incremental reads.
+    boolean incrementalRead = conf.get(FlinkOptions.READ_AS_STREAMING)
+        || 
FlinkOptions.QUERY_TYPE_INCREMENTAL.equals(conf.get(FlinkOptions.QUERY_TYPE));
+    this.partitionPruner = createPartitionPruner(splitFilters.f1, 
incrementalRead ? null : columnStatsProbe);
     this.dataBucketFunc = getDataBucketFunc(splitFilters.f0);
     // refuse all the filters now
     return SupportsFilterPushDown.Result.of(new ArrayList<>(splitFilters.f1), 
new ArrayList<>(filters));
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
index 9c8e0a7f7db1..1435f3029cd4 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
@@ -402,6 +402,7 @@ public class TestIncrementalInputSplits extends 
HoodieCommonTestHarness {
 
     PartitionPruners.PartitionPruner partitionPruner =
         
PartitionPruners.builder().rowType(TestConfigurations.ROW_TYPE).basePath(basePath).metaClient(metaClient).conf(conf).columnStatsProbe(columnStatsProbe).build();
+    assertEquals(Collections.singleton("par3"), 
partitionPruner.filter(Arrays.asList("par1", "par2", "par3", "par4")));
     IncrementalInputSplits iis = IncrementalInputSplits.builder()
         .conf(conf)
         .path(new Path(basePath))
@@ -410,7 +411,14 @@ public class TestIncrementalInputSplits extends 
HoodieCommonTestHarness {
         .build();
     IncrementalInputSplits.Result result = iis.inputSplits(metaClient, null, 
false);
     List<String> partitions = getFilteredPartitions(result);
-    assertEquals(Arrays.asList("par3"), partitions);
+    assertEquals(Collections.singletonList("par3"), partitions);
+
+    PartitionPruners.PartitionPruner chainedPruner = PartitionPruners.builder()
+        
.rowType(TestConfigurations.ROW_TYPE).basePath(basePath).metaClient(metaClient).conf(conf)
+        
.columnStatsProbe(columnStatsProbe).candidatePartitions(Arrays.asList("par1", 
"par3")).build();
+    IncrementalInputSplits chainedSplits = IncrementalInputSplits.builder()
+        .conf(conf).path(new 
Path(basePath)).rowType(TestConfigurations.ROW_TYPE).partitionPruner(chainedPruner).build();
+    assertEquals(Collections.singletonList("par3"), 
getFilteredPartitions(chainedSplits.inputSplits(metaClient, null, false)));
   }
 
   @Test
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index c877fda22097..09ba50f7edbc 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -2944,6 +2944,51 @@ public class ITTestHoodieDataSource {
             + "+I[id3, id3, Julian, 43, 1970-01-01T00:00:03, par1]]");
   }
 
+  @ParameterizedTest
+  @MethodSource("tableTypeAndSourceV2AndBooleanTrueFalseParams")
+  void testHistoricalIncrementalReadWithDataSkipping(HoodieTableType 
tableType, boolean useSourceV2, boolean streaming) throws Exception {
+    batchTableEnv.executeSql(sql("writer")
+        .option(FlinkOptions.PATH, tempFile.getAbsolutePath())
+        .options(getDefaultKeys())
+        .option(FlinkOptions.TABLE_TYPE, tableType)
+        .option(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true)
+        .option(FlinkOptions.METADATA_ENABLED, true)
+        .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), 
true)
+        
.option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), true)
+        .end());
+    execInsertSql(batchTableEnv, "insert into writer values "
+        + "('id1', 'alice', 23, TIMESTAMP '1970-01-01 00:00:01', 'par1'),"
+        + "('id2', 'bob', 23, TIMESTAMP '1970-01-01 00:00:01', 'par2')");
+    String firstCommit = 
HoodieTestUtils.createMetaClient(tempFile.getAbsolutePath())
+        
.getCommitsTimeline().filterCompletedInstants().lastInstant().get().getCompletionTime();
+    // The update can remove age 23 from current stats of par1; par2 remains a 
matching control.
+    execInsertSql(batchTableEnv, "insert into writer values ('id1', 'alice', 
50, TIMESTAMP '1970-01-01 00:00:02', 'par1')");
+    assertRowsEquals(execSelectSql(batchTableEnv, "select * from writer where 
age = 50"),
+        "[+I[id1, alice, 50, 1970-01-01T00:00:02, par1]]");
+
+    TableEnvironment readerEnv = streaming ? streamTableEnv : batchTableEnv;
+    readerEnv.executeSql(sql("t1")
+        .option(FlinkOptions.PATH, tempFile.getAbsolutePath())
+        .options(getDefaultKeys())
+        .option(FlinkOptions.TABLE_TYPE, tableType)
+        .option(FlinkOptions.READ_AS_STREAMING, streaming)
+        .option(FlinkOptions.READ_STREAMING_CHECK_INTERVAL, 2)
+        .option(FlinkOptions.READ_START_COMMIT, firstCommit)
+        .option(FlinkOptions.READ_END_COMMIT, firstCommit)
+        .option(FlinkOptions.READ_SOURCE_V2_ENABLED, useSourceV2)
+        .option(FlinkOptions.METADATA_ENABLED, true)
+        .option(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true)
+        .end());
+    String query = "select * from t1 where age = 23";
+    String expected = "[+I[id1, alice, 23, 1970-01-01T00:00:01, par1], "
+        + "+I[id2, bob, 23, 1970-01-01T00:00:01, par2]]";
+    if (streaming) {
+      assertRowsEquals(execSelectSqlWithExpectedNum(readerEnv, query, 2), 
expected);
+    } else {
+      assertRowsEquals(execSelectSql(readerEnv, query), expected);
+    }
+  }
+
   @Test
   void testDataSkippingWithMissingColumnStatsAfterIndexExpansion() throws 
Exception {
     TableEnvironment tableEnv = batchTableEnv;
@@ -4322,7 +4367,7 @@ public class ITTestHoodieDataSource {
   }
 
   /**
-   * Return test params => (HoodieTableType, true/false, true/fase).
+   * Return test params => (HoodieTableType, true/false, true/false).
    */
   private static Stream<Arguments> 
tableTypeAndSourceV2AndBooleanTrueFalseParams() {
     Object[][] data =
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
index f82c1267fd38..a91c436547ad 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
@@ -236,6 +236,33 @@ public class TestHoodieTableSource {
     assertEquals(expectedPartitions, hoodieTableSource.getReadPartitions());
   }
 
+  @ParameterizedTest
+  @ValueSource(strings = {"snapshot", "read_optimized", "incremental", 
"streaming"})
+  void testPartitionStatsPrunerForReadMode(String readMode) throws Exception {
+    Configuration conf = 
TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
+    conf.set(FlinkOptions.METADATA_ENABLED, true);
+    conf.set(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true);
+    
conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), 
"true");
+    
conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(),
 "true");
+    TestData.writeData(TestData.DATA_SET_INSERT, conf);
+    if ("streaming".equals(readMode)) {
+      conf.set(FlinkOptions.READ_AS_STREAMING, true);
+    } else {
+      conf.set(FlinkOptions.QUERY_TYPE, readMode);
+    }
+
+    HoodieTableSource tableSource = createHoodieTableSource(conf);
+    tableSource.applyFilters(Arrays.asList(
+        createLitEquivalenceExpr("age", 2, DataTypes.INT().notNull(), 18),
+        createLitEquivalenceExpr("partition", 4, DataTypes.STRING().notNull(), 
"par1")));
+    assertNotNull(tableSource.getColumnStatsProbe());
+    if ("incremental".equals(readMode) || "streaming".equals(readMode)) {
+      assertEquals(Collections.singletonList("par1"), 
tableSource.getReadPartitions());
+    } else {
+      assertTrue(tableSource.getReadPartitions().isEmpty());
+    }
+  }
+
   @Test
   void testDataSkippingWithMetaColumns() throws Exception {
     final String path = tempFile.getAbsolutePath();

Reply via email to