This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 295899cd1 [VL] Enable Spark mask function (#6271)
295899cd1 is described below
commit 295899cd108c56dda436730d58dd716d9d8b0b28
Author: 高阳阳 <[email protected]>
AuthorDate: Tue Jul 30 15:31:41 2024 +0800
[VL] Enable Spark mask function (#6271)
---
.../execution/ScalarFunctionsValidateSuite.scala | 18 ++++++++++++++++++
.../org/apache/gluten/expression/ExpressionNames.scala | 1 +
.../apache/gluten/sql/shims/spark34/Spark34Shims.scala | 3 ++-
.../apache/gluten/sql/shims/spark35/Spark35Shims.scala | 1 +
4 files changed, 22 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 43d54fb62..13ade14b5 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -918,6 +918,24 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ testWithSpecifiedSparkVersion("mask", Some("3.4")) {
+ runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT mask(c_comment, 'Y') FROM customer limit 50") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y') FROM customer limit
50") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o') FROM customer
limit 50") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT mask(c_comment, 'Y', 'y', 'o', '*') FROM
customer limit 50") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
test("bit_length") {
runQueryAndCompare(
"select bit_length(c_comment), bit_length(cast(c_comment as binary))" +
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 7cc75405b..d47dbc4cc 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -131,6 +131,7 @@ object ExpressionNames {
final val LEVENSHTEIN = "levenshtein"
final val UNBASE64 = "unbase64"
final val BASE64 = "base64"
+ final val MASK = "mask"
// URL functions
final val PARSE_URL = "parse_url"
diff --git
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index 203256cf5..cd7e4347d 100644
---
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -80,7 +80,8 @@ class Spark34Shims extends SparkShims {
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
Sig[RoundFloor](ExpressionNames.FLOOR),
- Sig[RoundCeil](ExpressionNames.CEIL)
+ Sig[RoundCeil](ExpressionNames.CEIL),
+ Sig[Mask](ExpressionNames.MASK)
)
}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 821e0f583..bb41b7e73 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -78,6 +78,7 @@ class Spark35Shims extends SparkShims {
Sig[Csc](ExpressionNames.CSC),
Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
+ Sig[Mask](ExpressionNames.MASK),
Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
