This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 83a3c0f79 [GLUTEN-5248][VL] Directly pass legacySizeOfNull to native
size function (#6014)
83a3c0f79 is described below
commit 83a3c0f79854f2cc855482da4a48ed9dcf333809
Author: PHILO-HE <[email protected]>
AuthorDate: Tue Jul 2 12:55:28 2024 +0800
[GLUTEN-5248][VL] Directly pass legacySizeOfNull to native size function
(#6014)
---
.../gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala | 7 -------
.../apache/gluten/expression/CHExpressionTransformer.scala | 10 ----------
.../gluten/execution/ScalarFunctionsValidateSuite.scala | 6 ++++++
.../scala/org/apache/gluten/execution/TestOperator.scala | 8 ++++----
.../org/apache/gluten/backendsapi/SparkPlanExecApi.scala | 7 -------
.../org/apache/gluten/expression/ExpressionConverter.scala | 12 +++++-------
6 files changed, 15 insertions(+), 35 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
index ac3ea61ff..7ed333aec 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala
@@ -612,13 +612,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr,
replaceExpr, original)
}
- override def genSizeExpressionTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: Size): ExpressionTransformer = {
- CHSizeExpressionTransformer(substraitExprName, child, original)
- }
-
override def genLikeTransformer(
substraitExprName: String,
left: ExpressionTransformer,
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala
index 5ca4e0233..7b389ead0 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala
@@ -30,16 +30,6 @@ import com.google.common.collect.Lists
import java.util.Locale
-case class CHSizeExpressionTransformer(
- substraitExprName: String,
- expr: ExpressionTransformer,
- original: Size)
- extends BinaryExpressionTransformer {
- override def left: ExpressionTransformer = expr
- // Pass legacyLiteral as second argument in substrait function
- override def right: ExpressionTransformer =
LiteralTransformer(original.legacySizeOfNull)
-}
-
case class CHTruncTimestampTransformer(
substraitExprName: String,
format: ExpressionTransformer,
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index bd32a799c..3db0f5e79 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -101,6 +101,12 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
+ testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) {
+ runQueryAndCompare("SELECT array_size(null)") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
test("chr function") {
val df = runQueryAndCompare(
"SELECT chr(l_orderkey + 64) " +
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
index d84f5e7cc..9b47a519c 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
@@ -1017,7 +1017,7 @@ class TestOperator extends
VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}
- ignore("test explode/posexplode function") {
+ test("test explode/posexplode function") {
Seq("explode", "posexplode").foreach {
func =>
// Literal: func(literal)
@@ -1190,7 +1190,7 @@ class TestOperator extends
VeloxWholeStageTransformerSuite with AdaptiveSparkPla
|""".stripMargin)(_)
}
- ignore("test multi-generate") {
+ test("test multi-generate") {
withTable("t") {
sql("CREATE TABLE t (col1 array<struct<a int, b string>>, col2
array<int>) using parquet")
sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')),
array(1, 2))")
@@ -1588,7 +1588,7 @@ class TestOperator extends
VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}
- ignore("test array literal") {
+ test("test array literal") {
withTable("array_table") {
sql("create table array_table(a array<bigint>) using parquet")
sql("insert into table array_table select array(1)")
@@ -1601,7 +1601,7 @@ class TestOperator extends
VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}
- ignore("test map literal") {
+ test("test map literal") {
withTable("map_table") {
sql("create table map_table(a map<bigint, string>) using parquet")
sql("insert into table map_table select map(1, 'hello')")
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
index 3ca5e0313..ff7449e2d 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
@@ -464,13 +464,6 @@ trait SparkPlanExecApi {
original)
}
- def genSizeExpressionTransformer(
- substraitExprName: String,
- child: ExpressionTransformer,
- original: Size): ExpressionTransformer = {
- GenericExpressionTransformer(substraitExprName, Seq(child), original)
- }
-
def genLikeTransformer(
substraitExprName: String,
left: ExpressionTransformer,
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index d5222cfc6..b5bcb6876 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -396,14 +396,12 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
r
)
case size: Size =>
- if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) {
- throw new GlutenNotSupportException(
- "The value of legacySizeOfNull field of size is " +
- "not equals to legacySizeOfNull of SQLConf, this case is not
supported yet")
- }
-
BackendsApiManager.getSparkPlanExecApiInstance.genSizeExpressionTransformer(
+ // Covers Spark ArraySize which is replaced by Size(child, false).
+ val child =
+ replaceWithExpressionTransformerInternal(size.child, attributeSeq,
expressionsMap)
+ GenericExpressionTransformer(
substraitExprName,
- replaceWithExpressionTransformerInternal(size.child, attributeSeq,
expressionsMap),
+ Seq(child, LiteralTransformer(size.legacySizeOfNull)),
size)
case namedStruct: CreateNamedStruct =>
BackendsApiManager.getSparkPlanExecApiInstance.genNamedStructTransformer(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
