This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ca9b6dadf4 [VL] Offload transform and filter lambdas that take an
index argument (#13600)
ca9b6dadf4 is described below
commit ca9b6dadf47e18624000b1737ebfd79a5a5cb583
Author: Pedrum Jalali <[email protected]>
AuthorDate: Tue Nov 10 12:37:11 2026 -0800
[VL] Offload transform and filter lambdas that take an index argument
(#13600)
---
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 24 +++-------------------
.../functions/ScalarFunctionsValidateSuite.scala | 10 ++++++++-
2 files changed, 12 insertions(+), 22 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 0377d23c72..087753c21f 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -216,9 +216,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
// Transformer for array_compact.
case LambdaFunction(_: IsNotNull, _, _) =>
GenericExpressionTransformer(ExpressionNames.ARRAY_COMPACT,
Seq(argument), expr)
- case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
- throw new GlutenNotSupportException(
- "filter on array with lambda using index argument is not supported
yet")
case _ => GenericExpressionTransformer(substraitExprName, Seq(argument,
function), expr)
}
}
@@ -229,12 +226,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
argument: ExpressionTransformer,
function: ExpressionTransformer,
expr: ArrayForAll): ExpressionTransformer = {
- expr.function match {
- case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
- throw new GlutenNotSupportException(
- "forall on array with lambda using index argument is not supported
yet")
- case _ => GenericExpressionTransformer(substraitExprName, Seq(argument,
function), expr)
- }
+ GenericExpressionTransformer(substraitExprName, Seq(argument, function),
expr)
}
override def genArraySortTransformer(
@@ -251,12 +243,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
argument: ExpressionTransformer,
function: ExpressionTransformer,
expr: ArrayExists): ExpressionTransformer = {
- expr.function match {
- case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
- throw new GlutenNotSupportException(
- "exists on array with lambda using index argument is not supported
yet")
- case _ => GenericExpressionTransformer(substraitExprName, Seq(argument,
function), expr)
- }
+ GenericExpressionTransformer(substraitExprName, Seq(argument, function),
expr)
}
/** Transform array transform to Substrait. */
@@ -265,12 +252,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
argument: ExpressionTransformer,
function: ExpressionTransformer,
expr: ArrayTransform): ExpressionTransformer = {
- expr.function match {
- case LambdaFunction(_, arguments, _) if arguments.size == 2 =>
- throw new GlutenNotSupportException(
- "transform on array with lambda using index argument is not
supported yet")
- case _ => GenericExpressionTransformer(substraitExprName, Seq(argument,
function), expr)
- }
+ GenericExpressionTransformer(substraitExprName, Seq(argument, function),
expr)
}
/** Transform posexplode to Substrait. */
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 2a2ade72f0..48bcd02e6b 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -1072,6 +1072,11 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
runQueryAndCompare("select filter(value, x -> x is not null) as res
from array_tbl;") {
checkGlutenPlan[ProjectExecTransformer]
}
+
+ runQueryAndCompare(
+ "select filter(value, (x, i) -> i < size(value) - 1) as res from
array_tbl;") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
}
}
@@ -1095,10 +1100,13 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
test("transform") {
withTable("t") {
sql("create table t (arr ARRAY<INT>) using parquet")
- sql("insert into t values(array(1, 2, 3, null))")
+ sql("insert into t values (array(1, 2, 3, null)), (array()), (null)")
runQueryAndCompare("select transform(arr, x -> x + 1) from t") {
checkGlutenPlan[ProjectExecTransformer]
}
+ runQueryAndCompare("select transform(arr, (x, i) -> x + i) from t") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]