This is an automated email from the ASF dual-hosted git repository.

philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new cabe274319 [VL][BOLT] Only rewrite filter(arr, x -> x IS NOT NULL) to 
array_compact (#13603)
cabe274319 is described below

commit cabe2743195ed02c815bcbc58109df1ff4a4e3d2
Author: Pedrum Jalali <[email protected]>
AuthorDate: Wed Nov 11 23:15:30 2026 -0800

    [VL][BOLT] Only rewrite filter(arr, x -> x IS NOT NULL) to array_compact 
(#13603)
---
 .../gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala   |  3 ++-
 .../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala |  3 ++-
 .../gluten/functions/ScalarFunctionsValidateSuite.scala  | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git 
a/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
 
b/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
index 12508f0e39..738325ef54 100644
--- 
a/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
+++ 
b/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
@@ -191,7 +191,8 @@ class BoltSparkPlanExecApi extends SparkPlanExecApi {
       expr: ArrayFilter): ExpressionTransformer = {
     expr.function match {
       // Transformer for array_compact.
-      case LambdaFunction(_: IsNotNull, _, _) =>
+      case LambdaFunction(IsNotNull(element: NamedLambdaVariable), 
Seq(variable), _)
+          if element.exprId == variable.exprId =>
         GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT, 
Seq(argument), expr)
       case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
         throw new GlutenNotSupportException(
diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index b8b3dce549..ce107b6f74 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -216,7 +216,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       expr: ArrayFilter): ExpressionTransformer = {
     expr.function match {
       // Transformer for array_compact.
-      case LambdaFunction(_: IsNotNull, _, _) =>
+      case LambdaFunction(IsNotNull(element: NamedLambdaVariable), 
Seq(variable), _)
+          if element.exprId == variable.exprId =>
         GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT, 
Seq(argument), expr)
       case _ => GenericExpressionTransformer(substraitExprName, Seq(argument, 
function), expr)
     }
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 48bcd02e6b..e92ff7c888 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -1080,6 +1080,22 @@ class ScalarFunctionsValidateSuite extends 
FunctionsValidateSuite {
     }
   }
 
+  test("filter with is not null on a struct field or an outer column") {
+    withTable("t") {
+      sql("create table t (arr array<struct<a: int>>, c int) using parquet")
+      sql(
+        "insert into t values " +
+          "(array(named_struct('a', 1), named_struct('a', cast(null as int))), 
null), " +
+          "(array(named_struct('a', 2), null), 3)")
+      runQueryAndCompare("select filter(arr, x -> x.a is not null) from t") {
+        checkGlutenPlan[ProjectExecTransformer]
+      }
+      runQueryAndCompare("select filter(arr, x -> c is not null) from t") {
+        checkGlutenPlan[ProjectExecTransformer]
+      }
+    }
+  }
+
   test("concat") {
     withTempPath {
       path =>


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to