Tan-JiaLiang commented on code in PR #6060:
URL: https://github.com/apache/paimon/pull/6060#discussion_r2281925969
##########
paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -792,23 +796,67 @@ public ColumnIndexStore getColumnIndexStore(int
blockIndex) {
}
private RowRanges getRowRanges(int blockIndex) {
- assert FilterCompat.isFilteringRequired(options.getRecordFilter())
- : "Should not be invoked if filter is null or NOOP";
+ boolean filteringRequired =
FilterCompat.isFilteringRequired(options.getRecordFilter());
+ if (!filteringRequired && selection == null) {
+ throw new IllegalArgumentException("Should not be invoked if
filter is null or NOOP");
+ }
+
RowRanges rowRanges = blockRowRanges.get(blockIndex);
if (rowRanges == null) {
- rowRanges =
- ColumnIndexFilter.calculateRowRanges(
- options.getRecordFilter(),
- getColumnIndexStore(blockIndex),
- paths.keySet(),
- blocks.get(blockIndex).getRowCount(),
- blocks.get(blockIndex).getRowIndexOffset(),
- selection);
+ BlockMetaData block = blocks.get(blockIndex);
+ rowRanges = RowRanges.createSingle(block.getRowCount());
+ if (selection != null) {
+ RowRanges result = calculateRowRanges(blockIndex, selection);
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
+
+ if (filteringRequired) {
+ RowRanges result =
+ ColumnIndexFilter.calculateRowRanges(
+ options.getRecordFilter(),
+ getColumnIndexStore(blockIndex),
+ paths.keySet(),
+ block.getRowCount());
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
blockRowRanges.set(blockIndex, rowRanges);
}
return rowRanges;
}
+ private RowRanges calculateRowRanges(int blockIndex, RoaringBitmap32
selection) {
+ List<OffsetIndex> offsets;
+ BlockMetaData block = blocks.get(blockIndex);
+ if (paths.isEmpty()) {
Review Comment:
I want to using the selection to let the limit pushdown to filter the pages
in the another PR.
If skip here, selection can not works in the following example, and it will
be fallback to read all pages.
```sql
SELECT COUNT(*) FROM T LIMIT 10;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
