alamb commented on code in PR #25460:
URL: https://github.com/apache/datafusion/pull/25460#discussion_r4081821787
##########
datafusion/datasource-parquet/src/opener/mod.rs:
##########
@@ -526,6 +525,65 @@ struct FiltersPreparedParquetOpen {
struct RowGroupsPrunedParquetOpen {
prepared: FiltersPreparedParquetOpen,
row_groups: RowGroupAccessPlanFilter,
+ /// Built lazily for external-selection index checks and reused by the
stream.
Review Comment:
❤️
##########
datafusion/datasource-parquet/src/opener/mod.rs:
##########
@@ -1572,17 +1692,15 @@ impl RowGroupsPrunedParquetOpen {
let arrow_reader_metrics = ArrowReaderMetrics::enabled();
- // Build the decoder projection (mask + per-batch transform) in a
- // single call. Encapsulating it behind `DecoderProjection` keeps the
- // opener's orchestration body focused on filter / decoder / stream
- // wiring.
- let decoder_projection = DecoderProjection::try_new(
- &prepared.projection,
- &prepared.physical_file_schema,
- reader_metadata.parquet_schema(),
- &prepared.output_schema,
- prepared.virtual_state.as_deref(),
- )?;
+ // Reuse plans built for the external-selection index check. Other
Review Comment:
this is great
##########
datafusion/datasource-parquet/src/opener/mod.rs:
##########
@@ -2282,8 +2396,9 @@ mod test {
page_pruning_predicate,
},
row_groups: RowGroupAccessPlanFilter::new(plan),
+ decoder_read_plans: None,
};
- open.should_load_page_index()
+ open.should_load_page_index().unwrap()
Review Comment:
how do we know this won't panic? Maybe this should also return Result?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
