xudong963 commented on code in PR #18868:
URL: https://github.com/apache/datafusion/pull/18868#discussion_r2549196593
##########
datafusion/datasource-parquet/src/row_group_filter.rs:
##########
@@ -135,13 +186,56 @@ impl RowGroupAccessPlanFilter {
// try to prune the row groups in a single call
match predicate.prune(&pruning_stats) {
Ok(values) => {
- // values[i] is false means the predicate could not be true
for row group i
+ let mut fully_contained_candidates_original_idx: Vec<usize> =
Vec::new();
for (idx, &value) in
row_group_indexes.iter().zip(values.iter()) {
if !value {
self.access_plan.skip(*idx);
metrics.row_groups_pruned_statistics.add_pruned(1);
} else {
metrics.row_groups_pruned_statistics.add_matched(1);
+ fully_contained_candidates_original_idx.push(*idx);
+ }
+ }
+
+ // Note: this part of code shouldn't be expensive with a
limited number of row groups
+ // If we do find it's expensive, we can consider optimizing it
further.
+ if !fully_contained_candidates_original_idx.is_empty() {
+ // Use NotExpr to create the inverted predicate
+ let inverted_expr =
+
Arc::new(NotExpr::new(Arc::clone(predicate.orig_expr())));
+ // Simplify the NOT expression (e.g., NOT(c1 = 0) -> c1 !=
0)
+ // before building the pruning predicate
+ let mut simplifier =
PhysicalExprSimplifier::new(arrow_schema);
Review Comment:
@adriangb that's why I opened the PR
https://github.com/apache/datafusion/pull/18587. Finally, you made me think
more and I put some logic into PhysicalExprSimplifier
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
