notashes commented on code in PR #20413:
URL: https://github.com/apache/datafusion/pull/20413#discussion_r2820838353
##########
datafusion/datasource-parquet/src/opener.rs:
##########
@@ -459,27 +465,55 @@ impl FileOpener for ParquetOpener {
// `row_filter` for details.
//
---------------------------------------------------------------------
- // Filter pushdown: evaluate predicates during scan
- if let Some(predicate) =
pushdown_filters.then_some(predicate).flatten() {
- let row_filter = row_filter::build_row_filter(
- &predicate,
- &physical_file_schema,
- builder.metadata(),
- reorder_predicates,
- &file_metrics,
- );
+ // When a dynamic filter is present, defer expensive string-column
+ // predicates to post-decode batch filtering to avoid blocking
+ // dynamic filter convergence via predicate cache overhead.
+ //
+ // Stats pruning still uses the full predicate (unaffected).
+ let mut leftover_filter: Option<Arc<dyn PhysicalExpr>> = None;
- match row_filter {
- Ok(Some(filter)) => {
- builder = builder.with_row_filter(filter);
- }
- Ok(None) => {}
- Err(e) => {
- debug!(
- "Ignoring error building row filter for
'{predicate:?}': {e}"
- );
- }
+ if let Some(predicate) =
pushdown_filters.then_some(predicate).flatten() {
+ let conjuncts = split_conjunction(&predicate);
+ let has_dynamic = conjuncts.iter().any(|c|
is_dynamic_physical_expr(c));
+
+ let (row_filter_pred, leftover_pred) = if has_dynamic {
+ let (deferred, rf_conjuncts): (Vec<_>, Vec<_>) =
+ conjuncts.into_iter().partition(|c| {
+ !is_dynamic_physical_expr(c)
Review Comment:
heya @Dandandan
with regards to the current discussions and #20417 do you think it's better
to close this PR?
regarding the page level pruning I saw that datafusions
`PagePruningAccessPlanFilter` seems to use the original unsplit predicate. And
how it's taken care of within arrow-rs I didn't get the time to dig into it
yet!
let me know what you think! thank you again!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
