This is an automated email from the ASF dual-hosted git repository.
zml1206 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f7f5f0433c [VL] Allow casting NULL constants to complex types (#12950)
f7f5f0433c is described below
commit f7f5f0433c6996644f9aad285053517a6a02f863
Author: kevinwilfong <[email protected]>
AuthorDate: Wed Sep 2 01:17:44 2026 -0700
[VL] Allow casting NULL constants to complex types (#12950)
---
.../gluten/execution/MiscOperatorSuite.scala | 30 ++++++++++++
.../substrait/SubstraitToVeloxPlanValidator.cc | 8 ++++
.../tests/Substrait2VeloxPlanValidatorTest.cc | 53 ++++++++++++++++++++++
.../gluten/utils/velox/VeloxTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 1 +
6 files changed, 96 insertions(+)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 6ddeaad2ef..948df8f8a3 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -22,6 +22,7 @@ import org.apache.gluten.expression.VeloxDummyExpression
import org.apache.spark.SparkConf
import org.apache.spark.shuffle.GlutenShuffleUtils
import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.catalyst.expressions.Cast
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper,
AQEShuffleReadExec, ColumnarAQEShuffleReadExec, ShuffleQueryStageExec}
import org.apache.spark.sql.execution.joins.BaseJoinExec
@@ -1957,6 +1958,35 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
}
+ test("cast null type to complex type") {
+ // An outer join whose right side turns out to be empty is replaced with a
projection of a
+ // null cast to the type of each of that side's output attributes. Here
the right side is only
+ // known to be empty once its stage has run, so AQE adds the casts after
constant folding and
+ // they reach the backend as casts from the null type rather than as typed
null literals.
+ val query =
+ """
+ |select l.l_orderkey, r.arr, r.m, r.s
+ |from lineitem l left outer join (
+ | select l_orderkey, array(l_partkey) as arr, map('k', l_partkey) as
m,
+ | struct(l_partkey as a) as s
+ | from lineitem where l_orderkey < 0
+ |) r on l.l_orderkey = r.l_orderkey
+ |""".stripMargin
+ runQueryAndCompare(query) {
+ df =>
+ val plan = df.queryExecution.executedPlan
+ val castsToComplexTypes = collect(plan) { case p:
ProjectExecTransformer => p }
+ .flatMap(_.projectList)
+ .flatMap(_.collect { case c: Cast if c.child.dataType == NullType =>
c.dataType })
+ assert(
+ castsToComplexTypes.exists(_.isInstanceOf[ArrayType]),
+ s"Expect the null casts to be offloaded in:\n$plan")
+ // The casts must run natively rather than being split out to the JVM.
+ assert(collect(plan) { case p: ColumnarPartialProjectExec => p
}.isEmpty)
+ assert(collect(plan) { case p: ProjectExec => p }.isEmpty)
+ }
+ }
+
test("timestamp broadcast join") {
spark.range(0, 5).createOrReplaceTempView("right")
spark.sql("SELECT id, timestamp_micros(id) as ts from
right").createOrReplaceTempView("left")
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 9896b43e54..2b36c1ff41 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -247,6 +247,14 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const
TypePtr& fromType, const
return false;
}
+ // Casting from UNKNOWN, e.g. a null constant, is allowed for any target
type,
+ // including complex ones. The input is all nulls, so Velox short-circuits
the
+ // cast to a null constant of the target type without ever looking at the
+ // input values.
+ if (fromType->kind() == TypeKind::UNKNOWN) {
+ return true;
+ }
+
// Limited support for DATE to X.
if (fromType->isDate() && !toType->isTimestamp() && !toType->isVarchar()) {
return false;
diff --git a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc
b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc
index a1147be798..c8c41e4e16 100644
--- a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc
+++ b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc
@@ -150,4 +150,57 @@ TEST_F(Substrait2VeloxPlanValidatorTest,
aggregateMaskMustBeTopLevelField) {
EXPECT_FALSE(validatePlan(nestedPlan));
}
+TEST_F(Substrait2VeloxPlanValidatorTest, castFromUnknown) {
+ const auto validateCast = [&](const RowTypePtr& inputType,
+ const
std::function<void(::substrait::Expression*)>& setInput,
+ const std::function<void(::substrait::Type*)>&
setToType) {
+ ::substrait::Expression expression;
+ auto* cast = expression.mutable_cast();
+ setInput(cast->mutable_input());
+ setToType(cast->mutable_type());
+
+ auto planValidator =
std::make_shared<SubstraitToVeloxPlanValidator>(pool_.get());
+ return planValidator->validate(expression, inputType, {});
+ };
+
+ // A null constant, e.g. Spark's NullType, is expressed as the Nothing type.
+ const auto setNullInput = [](::substrait::Expression* input) {
+ input->mutable_literal()->mutable_null()->mutable_nothing();
+ };
+ const auto setNullableI32 = [](::substrait::Type* type) {
+
type->mutable_i32()->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ };
+
+ // Casting from Nothing is allowed for both scalar and complex target types.
+ EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, setNullableI32));
+
+ EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type*
type) {
+ auto* list = type->mutable_list();
+ list->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ setNullableI32(list->mutable_type());
+ }));
+
+ EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type*
type) {
+ auto* map = type->mutable_map();
+ map->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ setNullableI32(map->mutable_key());
+ setNullableI32(map->mutable_value());
+ }));
+
+ EXPECT_TRUE(validateCast(ROW({}, {}), setNullInput, [&](::substrait::Type*
type) {
+ auto* structType = type->mutable_struct_();
+
structType->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
+ structType->add_names("");
+ setNullableI32(structType->add_types());
+ }));
+
+ // Casting a complex type to an unrelated type is still not allowed.
+ EXPECT_FALSE(validateCast(
+ ROW({"a"}, {ARRAY(INTEGER())}),
+ [](::substrait::Expression* input) {
+
input->mutable_selection()->mutable_direct_reference()->mutable_struct_field()->set_field(0);
+ },
+ setNullableI32));
+}
+
} // namespace gluten
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 538ca28417..8e8ea39605 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -942,6 +942,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for atomic top-level fields (byPosition)")
.exclude("NOT NULL checks for nested struct fields (byName)")
.exclude("NOT NULL checks for nested struct fields (byPosition)")
+ .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
+ .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
.exclude("not null checks for fields inside nullable array (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 053369d2be..9841eade00 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1225,6 +1225,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for atomic top-level fields (byPosition)")
.exclude("NOT NULL checks for nested struct fields (byName)")
.exclude("NOT NULL checks for nested struct fields (byPosition)")
+ .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
+ .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
.exclude("not null checks for fields inside nullable array (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 61f202afdd..d2bd2832ab 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1223,6 +1223,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for nested struct fields (byName)")
.exclude("NOT NULL checks for nested struct fields (byPosition)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
+ .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
.exclude("not null checks for fields inside nullable array (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
