This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 730ca45eef [GLUTEN-9015][VL] Support array_append function (#9016)
730ca45eef is described below
commit 730ca45eeffd3109475c914b20ffd23fbf266da4
Author: Qian Sun <[email protected]>
AuthorDate: Tue Mar 18 16:07:38 2025 +0800
[GLUTEN-9015][VL] Support array_append function (#9016)
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 3 +-
.../execution/ScalarFunctionsValidateSuite.scala | 49 ++++++++++++++++++++++
.../apache/gluten/expression/ExpressionNames.scala | 1 +
.../gluten/sql/shims/spark34/Spark34Shims.scala | 3 +-
.../gluten/sql/shims/spark35/Spark35Shims.scala | 3 +-
5 files changed, 56 insertions(+), 3 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 6e3d2ec19a..2728b8c412 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -205,6 +205,7 @@ object CHExpressionUtil {
RAISE_ERROR -> DefaultValidator(),
WIDTH_BUCKET -> DefaultValidator(),
MAKE_DATE -> DefaultValidator(),
- MAP_CONCAT -> DefaultValidator()
+ MAP_CONCAT -> DefaultValidator(),
+ ARRAY_APPEND -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 3b5865a3a3..4821f4f9f5 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -123,6 +123,55 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ testWithSpecifiedSparkVersion("Test array_append function - INT",
Some("3.4")) {
+ withTempPath {
+ path =>
+ Seq[(Array[Int], Int)](
+ (Array(2, 1), 0),
+ (Array(1), 1),
+ (Array(), 0),
+ (Array(1, 2, null.asInstanceOf[Int]), 1),
+ (Array(null.asInstanceOf[Int]), 1),
+ (Array(null.asInstanceOf[Int]), null.asInstanceOf[Int]),
+ (Array(), null.asInstanceOf[Int]),
+ (null.asInstanceOf[Array[Int]], 1)
+ )
+ .toDF("arr", "num")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("select arr, num, array_append(arr, num) from tbl")
{
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
+ testWithSpecifiedSparkVersion("Test array_append function - STRING",
Some("3.4")) {
+ withTempPath {
+ path =>
+ Seq[(Array[String], String)](
+ (Array("a", "b"), "c"),
+ (Array("a"), "b"),
+ (Array(), "a"),
+ (Array("a", "b", null.asInstanceOf[String]), "c"),
+ (Array(null.asInstanceOf[String]), "a"),
+ (Array(null.asInstanceOf[String]), null.asInstanceOf[String]),
+ (Array(), null.asInstanceOf[String])
+ )
+ .toDF("arr", "txt")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("select arr, txt, array_append(arr, txt) from tbl")
{
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("Test round function") {
runQueryAndCompare(
"SELECT round(cast(l_orderkey as int), 2)" +
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 148933e142..6c85aec447 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -278,6 +278,7 @@ object ExpressionNames {
final val ZIP_WITH = "zip_with"
final val FLATTEN = "flatten"
final val ARRAY_INSERT = "array_insert"
+ final val ARRAY_APPEND = "array_append"
// Map functions
final val CREATE_MAP = "map"
diff --git
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index f230bf7375..123a1c7da0 100644
---
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -88,7 +88,8 @@ class Spark34Shims extends SparkShims {
Sig[RoundCeil](ExpressionNames.CEIL),
Sig[Mask](ExpressionNames.MASK),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
-
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT)
+
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
+ Sig[ArrayAppend](ExpressionNames.ARRAY_APPEND)
)
}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 2203e08d3e..67d89e2635 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -91,7 +91,8 @@ class Spark35Shims extends SparkShims {
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
-
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT)
+
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
+ Sig[ArrayAppend](ExpressionNames.ARRAY_APPEND)
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
