This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cabe274319 [VL][BOLT] Only rewrite filter(arr, x -> x IS NOT NULL) to
array_compact (#13603)
cabe274319 is described below
commit cabe2743195ed02c815bcbc58109df1ff4a4e3d2
Author: Pedrum Jalali <[email protected]>
AuthorDate: Wed Nov 11 23:15:30 2026 -0800
[VL][BOLT] Only rewrite filter(arr, x -> x IS NOT NULL) to array_compact
(#13603)
---
.../gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala | 3 ++-
.../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala | 3 ++-
.../gluten/functions/ScalarFunctionsValidateSuite.scala | 16 ++++++++++++++++
3 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
b/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
index 12508f0e39..738325ef54 100644
---
a/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
+++
b/backends-bolt/src/main/scala/org/apache/gluten/backendsapi/bolt/BoltSparkPlanExecApi.scala
@@ -191,7 +191,8 @@ class BoltSparkPlanExecApi extends SparkPlanExecApi {
expr: ArrayFilter): ExpressionTransformer = {
expr.function match {
// Transformer for array_compact.
- case LambdaFunction(_: IsNotNull, _, _) =>
+ case LambdaFunction(IsNotNull(element: NamedLambdaVariable),
Seq(variable), _)
+ if element.exprId == variable.exprId =>
GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT,
Seq(argument), expr)
case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
throw new GlutenNotSupportException(
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index b8b3dce549..ce107b6f74 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -216,7 +216,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
expr: ArrayFilter): ExpressionTransformer = {
expr.function match {
// Transformer for array_compact.
- case LambdaFunction(_: IsNotNull, _, _) =>
+ case LambdaFunction(IsNotNull(element: NamedLambdaVariable),
Seq(variable), _)
+ if element.exprId == variable.exprId =>
GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT,
Seq(argument), expr)
case _ => GenericExpressionTransformer(substraitExprName, Seq(argument,
function), expr)
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 48bcd02e6b..e92ff7c888 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -1080,6 +1080,22 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("filter with is not null on a struct field or an outer column") {
+ withTable("t") {
+ sql("create table t (arr array<struct<a: int>>, c int) using parquet")
+ sql(
+ "insert into t values " +
+ "(array(named_struct('a', 1), named_struct('a', cast(null as int))),
null), " +
+ "(array(named_struct('a', 2), null), 3)")
+ runQueryAndCompare("select filter(arr, x -> x.a is not null) from t") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select filter(arr, x -> c is not null) from t") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+
test("concat") {
withTempPath {
path =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]