[arrow-datafusion] branch master updated: apply type coercion before filter pushdown (#3459)

[alamb]

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/master by this push:
     new af0d50a3c apply type coercion before filter pushdown (#3459)
af0d50a3c is described below

commit af0d50a3c393e3db99d6c1c9e1e9b06a19345853
Author: Kun Liu <liu...@apache.org>
AuthorDate: Tue Sep 13 19:05:31 2022 +0800

    apply type coercion before filter pushdown (#3459)
---
 datafusion/core/src/execution/context.rs | 4 +---
 datafusion/core/tests/parquet_pruning.rs | 3 +--
 datafusion/core/tests/sql/subqueries.rs  | 4 ++--
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/datafusion/core/src/execution/context.rs 
b/datafusion/core/src/execution/context.rs
index 64f75dfa2..9540ca7ad 100644
--- a/datafusion/core/src/execution/context.rs
+++ b/datafusion/core/src/execution/context.rs
@@ -1448,10 +1448,8 @@ impl SessionState {
             rules.push(Arc::new(FilterNullJoinKeys::default()));
         }
         rules.push(Arc::new(ReduceOuterJoin::new()));
-        rules.push(Arc::new(FilterPushDown::new()));
-        // we do type coercion after filter push down so that we don't push 
CAST filters to Parquet
-        // until https://github.com/apache/arrow-datafusion/issues/3289 is 
resolved
         rules.push(Arc::new(TypeCoercion::new()));
+        rules.push(Arc::new(FilterPushDown::new()));
         rules.push(Arc::new(LimitPushDown::new()));
         rules.push(Arc::new(SingleDistinctToGroupBy::new()));
 
diff --git a/datafusion/core/tests/parquet_pruning.rs 
b/datafusion/core/tests/parquet_pruning.rs
index 6681a748e..1de1d7c8f 100644
--- a/datafusion/core/tests/parquet_pruning.rs
+++ b/datafusion/core/tests/parquet_pruning.rs
@@ -625,8 +625,7 @@ impl ContextWithParquet {
             .sql(sql)
             .await
             .expect("planning")
-            .to_logical_plan()
-            .unwrap();
+            .to_unoptimized_plan();
         self.run_test(logical_plan, sql).await
     }
 
diff --git a/datafusion/core/tests/sql/subqueries.rs 
b/datafusion/core/tests/sql/subqueries.rs
index 1ae5bc68e..0ac286d76 100644
--- a/datafusion/core/tests/sql/subqueries.rs
+++ b/datafusion/core/tests/sql/subqueries.rs
@@ -339,7 +339,7 @@ order by s_name;
             Projection: #lineitem.l_partkey, #lineitem.l_suppkey, 
Decimal128(Some(50000000000000000),38,17) * CAST(#SUM(lineitem.l_quantity) AS 
Decimal128(38, 17)) AS __value, alias=__sq_3
               Aggregate: groupBy=[[#lineitem.l_partkey, #lineitem.l_suppkey]], 
aggr=[[SUM(#lineitem.l_quantity)]]
                 Filter: #lineitem.l_shipdate >= Date32("8766")
-                  TableScan: lineitem projection=[l_partkey, l_suppkey, 
l_quantity, l_shipdate], partial_filters=[#lineitem.l_shipdate >= 
CAST(Utf8("1994-01-01") AS Date32)]"#
+                  TableScan: lineitem projection=[l_partkey, l_suppkey, 
l_quantity, l_shipdate], partial_filters=[#lineitem.l_shipdate >= 
Date32("8766")]"#
         .to_string();
     assert_eq!(actual, expected);
 
@@ -394,7 +394,7 @@ order by cntrycode;"#;
               Projection: #AVG(customer.c_acctbal) AS __value, alias=__sq_1
                 Aggregate: groupBy=[[]], aggr=[[AVG(#customer.c_acctbal)]]
                   Filter: CAST(#customer.c_acctbal AS Decimal128(30, 15)) > 
Decimal128(Some(0),30,15) AND substr(#customer.c_phone, Int64(1), Int64(2)) IN 
([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), 
Utf8("17")])
-                    TableScan: customer projection=[c_phone, c_acctbal], 
partial_filters=[#customer.c_acctbal > Float64(0), substr(#customer.c_phone, 
Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), 
Utf8("30"), Utf8("18"), Utf8("17")])]"#
+                    TableScan: customer projection=[c_phone, c_acctbal], 
partial_filters=[CAST(#customer.c_acctbal AS Decimal128(30, 15)) > 
Decimal128(Some(0),30,15), substr(#customer.c_phone, Int64(1), Int64(2)) IN 
([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), 
Utf8("17")])]"#
         .to_string();
     assert_eq!(actual, expected);