alamb commented on code in PR #11157:
URL: https://github.com/apache/arrow-rs/pull/11157#discussion_r4134541674
##########
parquet/src/arrow/arrow_reader/mod.rs:
##########
@@ -774,6 +776,47 @@ impl ArrowReaderOptions {
self
}
+ /// Sets the same [`ColumnChunkMask`] for both page-index structures.
+ pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self {
+ self.with_column_index_mask(mask.clone())
+ .with_offset_index_mask(mask)
+ }
+
+ /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure.
Review Comment:
this is a nice API 👍
##########
parquet/src/file/metadata/push_decoder.rs:
##########
@@ -482,21 +519,38 @@ enum DecodeState {
}
/// Returns the byte range needed to read the offset/page indexes, based on the
-/// specified policies
+/// specified policies and masks
///
/// Returns None if no page indexes are needed
-pub fn range_for_page_index(
+pub(crate) fn range_for_page_index(
metadata: &ParquetMetaData,
column_index_policy: PageIndexPolicy,
offset_index_policy: PageIndexPolicy,
+ column_index_mask: &ColumnChunkMask,
+ offset_index_mask: &ColumnChunkMask,
) -> Option<Range<u64>> {
- let mut range = None;
- for c in metadata.row_groups().iter().flat_map(|r| r.columns()) {
- if column_index_policy != PageIndexPolicy::Skip {
- range = acc_range(range, c.column_index_range());
- }
- if offset_index_policy != PageIndexPolicy::Skip {
- range = acc_range(range, c.offset_index_range());
+ let mut range: Option<Range<u64>> = None;
+ for (policy, mask, column_index) in [
+ (column_index_policy, column_index_mask, true),
+ (offset_index_policy, offset_index_mask, false),
+ ] {
+ if policy != PageIndexPolicy::Skip {
+ for row_group in mask.row_group_indices(metadata.num_row_groups())
{
Review Comment:
nit is you can could reduce the indent with something like
```rust
if policy == PageIndexPolicy::Skip {
continue;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
