This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 84ffc3a8bb [core] Avoid unnecessary deletion vector I/O for empty
selections (#10470)
84ffc3a8bb is described below
commit 84ffc3a8bb0d3da80f7fc80d40fd1fa85caa6368
Author: sanshi <[email protected]>
AuthorDate: Fri Oct 9 21:53:01 2026 +0800
[core] Avoid unnecessary deletion vector I/O for empty selections (#10470)
---
.../paimon/operation/DataEvolutionSplitRead.java | 28 ++++++++++-
.../paimon/table/DataEvolutionFileIndexTest.java | 57 ++++++++++++++++++++++
2 files changed, 83 insertions(+), 2 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
index b053834633..ae2fc03d54 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
@@ -242,6 +242,9 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
if (fileIndexResult != null &&
!fileIndexResult.remain()) {
return new EmptyFileRecordReader<>();
}
+ if (isSelectionEmpty(file, rowRanges,
fileIndexResult)) {
+ return new EmptyFileRecordReader<>();
+ }
DeletionVectorWithRange deletionVector =
readDeletionVector(needMergeFiles,
deletionVectorFactory);
if (fileIndexResult != null
@@ -283,6 +286,12 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
.anyMatch(entry ->
!entry.result.remain())) {
return new EmptyFileRecordReader<>();
}
+ BitmapIndexResult groupSelection =
+ buildGroupSelection(needMergeFiles,
fileIndexResults);
+ if (isSelectionEmpty(
+ needMergeFiles.get(0), rowRanges,
groupSelection)) {
+ return new EmptyFileRecordReader<>();
+ }
DeletionVectorWithRange deletionVector =
readDeletionVector(needMergeFiles,
deletionVectorFactory);
if (deletionVector != null
@@ -293,9 +302,9 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
if (fileIndexResults == null) {
return new EmptyFileRecordReader<>();
}
+ groupSelection =
+ buildGroupSelection(needMergeFiles,
fileIndexResults);
}
- BitmapIndexResult groupSelection =
- buildGroupSelection(needMergeFiles,
fileIndexResults);
return createUnionReader(
needMergeFiles,
partition,
@@ -1049,6 +1058,21 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
return selection;
}
+ /** Returns whether exact bitmap candidates and the requested row ranges
are disjoint. */
+ private static boolean isSelectionEmpty(
+ DataFileMeta file,
+ @Nullable List<Range> rowRanges,
+ @Nullable FileIndexResult fileIndexResult) {
+ if (!(fileIndexResult instanceof BitmapIndexResult)) {
+ return false;
+ }
+ RoaringBitmap32 indexSelection = ((BitmapIndexResult)
fileIndexResult).get();
+ RoaringBitmap32 rangeSelection = file.toFileSelection(rowRanges);
+ return rangeSelection == null
+ ? indexSelection.isEmpty()
+ : RoaringBitmap32.and(rangeSelection,
indexSelection).isEmpty();
+ }
+
private boolean canPushDownGroupSelection(List<DataFileMeta> files) {
if (files.isEmpty()) {
return false;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index c2629f4f98..be205c1e3a 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -62,6 +62,7 @@ import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Range;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -583,6 +584,34 @@ public class DataEvolutionFileIndexTest extends
DataEvolutionTestBase {
assertThat(readWithFilter(table, equalF1(MISSING_F1))).isEmpty();
}
+ @Test
+ public void testSingleFileEmptySelectionSkipsBeforeReadingDeletionVector()
throws Exception {
+ Map<String, String> options = bitmapOptions("f1");
+ options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
+ FileStoreTable table = createTable("single_empty_selection_before_dv",
options);
+ writeAllColumns(table, ROW_COUNT);
+ deleteRows(table, 75);
+
+ FileStoreTable latest = getTable(identifier(table.name()));
+ DataSplit split = (DataSplit)
latest.newReadBuilder().newScan().plan().splits().get(0);
+ Path deletionVectorPath =
+ split.deletionFiles().get().stream()
+ .filter(Objects::nonNull)
+ .map(file -> new Path(file.path()))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new);
+ assertThat(latest.fileIO().delete(deletionVectorPath, false)).isTrue();
+
+ ReadBuilder readBuilder =
+ latest.newReadBuilder()
+ .withFilter(equalF1(f1(50)))
+ .withRowRanges(Collections.singletonList(new Range(0L,
49L)));
+ TableScan.Plan plan = readBuilder.newScan().plan();
+ assertThat(plan.splits()).hasSize(1);
+ assertThat(collect(readBuilder.newRead().executeFilter(), plan,
latest.rowType()))
+ .isEmpty();
+ }
+
@Test
public void testMergedGroupKeptWhenFilterColumnOverwritten() throws
Exception {
FileStoreTable table = createTable("overwritten",
Collections.emptyMap());
@@ -701,6 +730,34 @@ public class DataEvolutionFileIndexTest extends
DataEvolutionTestBase {
assertThat(readWithFilter(table, equalF1(MISSING_F1))).isEmpty();
}
+ @Test
+ public void
testMergedGroupEmptySelectionSkipsBeforeReadingDeletionVector() throws
Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
+ FileStoreTable table = createTable("merged_empty_selection_before_dv",
options);
+ writeSplitColumns(table, ROW_COUNT, bitmapOptions("f1"),
Collections.emptyMap());
+ deleteRows(table, 75);
+
+ FileStoreTable latest = getTable(identifier(table.name()));
+ DataSplit split = (DataSplit)
latest.newReadBuilder().newScan().plan().splits().get(0);
+ Path deletionVectorPath =
+ split.deletionFiles().get().stream()
+ .filter(Objects::nonNull)
+ .map(file -> new Path(file.path()))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new);
+ assertThat(latest.fileIO().delete(deletionVectorPath, false)).isTrue();
+
+ ReadBuilder readBuilder =
+ latest.newReadBuilder()
+ .withFilter(equalF1(f1(50)))
+ .withRowRanges(Collections.singletonList(new Range(0L,
49L)));
+ TableScan.Plan plan = readBuilder.newScan().plan();
+ assertThat(plan.splits()).hasSize(1);
+ assertThat(collect(readBuilder.newRead().executeFilter(), plan,
latest.rowType()))
+ .isEmpty();
+ }
+
/** Commits a deletion vector for the anchor file of the only row id group
of {@code table}. */
private void deleteRows(FileStoreTable table, long... positions) throws
Exception {
FileStoreTable latest = getTable(identifier(table.name()));
