This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 21e456578d [GLUTEN-9361][VL] Enable luhn_check function for spark 3.5
(#9665)
21e456578d is described below
commit 21e456578d820a85c6c1bcd8db31adfb7971b520
Author: dcoliversun <[email protected]>
AuthorDate: Mon May 19 00:16:07 2025 +0800
[GLUTEN-9361][VL] Enable luhn_check function for spark 3.5 (#9665)
---
.../src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++-
.../org/apache/gluten/execution/VeloxStringFunctionsSuite.scala | 6 ++++++
.../scala/org/apache/gluten/expression/ExpressionConverter.scala | 5 +++++
.../main/scala/org/apache/gluten/expression/ExpressionNames.scala | 1 +
.../scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala | 3 ++-
5 files changed, 16 insertions(+), 2 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 9cad2316f5..34d70b17c4 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -207,6 +207,7 @@ object CHExpressionUtil {
MAKE_DATE -> DefaultValidator(),
MAP_CONCAT -> DefaultValidator(),
ARRAY_APPEND -> DefaultValidator(),
- JSON_OBJECT_KEYS -> DefaultValidator()
+ JSON_OBJECT_KEYS -> DefaultValidator(),
+ LUHN_CHECK -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
index b95672a134..ec3a6079fe 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
@@ -665,4 +665,10 @@ class VeloxStringFunctionsSuite extends
VeloxWholeStageTransformerSuite {
s"select l_orderkey, right(l_comment, $NULL_STR_COL) " +
s"from $LINEITEM_TABLE limit
$LENGTH")(checkGlutenOperatorMatch[ProjectExecTransformer])
}
+
+ testWithMinSparkVersion("luhn_check", "3.5") {
+ runQueryAndCompare(
+ s"select l_orderkey, luhn_check(l_comment) " +
+ s"from $LINEITEM_TABLE limit
$LENGTH")(checkGlutenOperatorMatch[ProjectExecTransformer])
+ }
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index 6f2c9926d4..26dcf90124 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -167,6 +167,11 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
"url_" + i.functionName,
replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap),
i)
+ case i @ StaticInvoke(_, _, "isLuhnNumber", _, _, _, _, _) =>
+ return GenericExpressionTransformer(
+ ExpressionNames.LUHN_CHECK,
+ replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap),
+ i)
case StaticInvoke(clz, _, functionName, _, _, _, _, _) =>
throw new GlutenNotSupportException(
s"Not supported to transform StaticInvoke with object:
${clz.getName}, " +
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index c822161ea5..18d2856ca1 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -138,6 +138,7 @@ object ExpressionNames {
final val BASE64 = "base64"
final val MASK = "mask"
final val FORMAT_STRING = "format_string"
+ final val LUHN_CHECK = "luhn_check"
// URL functions
final val PARSE_URL = "parse_url"
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 543bef6c55..badf1ca66a 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -115,7 +115,8 @@ class Spark35Shims extends SparkShims {
Sig[EqualNull](ExpressionNames.EQUAL_NULL),
Sig[ILike](ExpressionNames.ILIKE),
Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY),
- Sig[Get](ExpressionNames.GET)
+ Sig[Get](ExpressionNames.GET),
+ Sig[Luhncheck](ExpressionNames.LUHN_CHECK)
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
