JingsongLi commented on code in PR #6060:
URL: https://github.com/apache/paimon/pull/6060#discussion_r2282109367
##########
paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -792,23 +796,67 @@ public ColumnIndexStore getColumnIndexStore(int
blockIndex) {
}
private RowRanges getRowRanges(int blockIndex) {
- assert FilterCompat.isFilteringRequired(options.getRecordFilter())
- : "Should not be invoked if filter is null or NOOP";
+ boolean filteringRequired =
FilterCompat.isFilteringRequired(options.getRecordFilter());
+ if (!filteringRequired && selection == null) {
+ throw new IllegalArgumentException("Should not be invoked if
filter is null or NOOP");
+ }
+
RowRanges rowRanges = blockRowRanges.get(blockIndex);
if (rowRanges == null) {
- rowRanges =
- ColumnIndexFilter.calculateRowRanges(
- options.getRecordFilter(),
- getColumnIndexStore(blockIndex),
- paths.keySet(),
- blocks.get(blockIndex).getRowCount(),
- blocks.get(blockIndex).getRowIndexOffset(),
- selection);
+ BlockMetaData block = blocks.get(blockIndex);
+ rowRanges = RowRanges.createSingle(block.getRowCount());
+ if (selection != null) {
+ RowRanges result = calculateRowRanges(blockIndex, selection);
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
+
+ if (filteringRequired) {
+ RowRanges result =
+ ColumnIndexFilter.calculateRowRanges(
+ options.getRecordFilter(),
+ getColumnIndexStore(blockIndex),
+ paths.keySet(),
+ block.getRowCount());
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
blockRowRanges.set(blockIndex, rowRanges);
}
return rowRanges;
}
+ private RowRanges calculateRowRanges(int blockIndex, RoaringBitmap32
selection) {
+ List<OffsetIndex> offsets;
+ BlockMetaData block = blocks.get(blockIndex);
+ if (paths.isEmpty()) {
Review Comment:
Let's finish this in another PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
