This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 01e71ec06b [GLUTEN-6625][VL] Support null elements in complex-type
literals (#13035)
01e71ec06b is described below
commit 01e71ec06b814c7b63af0306c24d9974bd2a30e8
Author: Kai Jiang <[email protected]>
AuthorDate: Sat Nov 7 17:39:04 2026 -0800
[GLUTEN-6625][VL] Support null elements in complex-type literals (#13035)
---
.../gluten/execution/VeloxLiteralSuite.scala | 27 +++-
cpp/velox/substrait/SubstraitToVeloxExpr.cc | 115 ++++++++++++----
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ++
.../sql/GlutenRuntimeNullChecksV2Writes.scala | 146 ++++++++++++++++++---
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ++
.../sql/GlutenRuntimeNullChecksV2Writes.scala | 144 +++++++++++++++++---
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ++
.../sql/GlutenRuntimeNullChecksV2Writes.scala | 144 +++++++++++++++++---
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ++
.../sql/GlutenRuntimeNullChecksV2Writes.scala | 144 +++++++++++++++++---
10 files changed, 642 insertions(+), 110 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
index cf2e7257f5..63a90abef3 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
@@ -111,14 +111,27 @@ class VeloxLiteralSuite extends
VeloxWholeStageTransformerSuite {
test("Null Literal") {
validateOffloadResult("SELECT cast(null as int)")
validateOffloadResult("SELECT cast(null as decimal)")
+ validateOffloadResult("SELECT CAST(null as struct<u:integer,v:string>)")
validateOffloadResult("SELECT array(5, 1, null)")
validateOffloadResult("SELECT array(5.321E2BD, 0.1, null)")
validateOffloadResult("SELECT struct('Spark', cast(null as int))")
validateOffloadResult("SELECT struct(cast(null as decimal))")
validateOffloadResult("SELECT map('b', 'a', 'e', null)")
- validateOffloadResult("SELECT array(null)")
validateOffloadResult("SELECT array(cast(null as int))")
validateOffloadResult("SELECT map(1, null)")
+ validateOffloadResult("SELECT array(null)")
+
+ validateOffloadResult("SELECT array(array(1,2), null, array(1), null)")
+ validateOffloadResult("SELECT array(array(1.0,2), null, array(1), null)")
+ validateOffloadResult("SELECT array(array(1, 2), null)")
+
+ validateOffloadResult("SELECT array(null, 2, 3.0)")
+ validateOffloadResult("SELECT array(1, 0, null, 2, 3)")
+ validateOffloadResult("SELECT array(null, array(null, array(1,2),
array(2)), null)")
+ validateOffloadResult("SELECT array(null, array(1, 2))")
+ validateOffloadResult("SELECT array(null, array(null, 1, 2))")
+ validateOffloadResult("SELECT array(null, array(1, 2, null), null)")
+ validateOffloadResult("SELECT array(null, array(null, array(1, 2), null,
array(1)))")
}
test("Scalar Type Literal") {
@@ -133,10 +146,14 @@ class VeloxLiteralSuite extends
VeloxWholeStageTransformerSuite {
validateOffloadResult("SELECT TIMESTAMP'2020-12-31',
TIMESTAMP'2020-12-30'")
validateOffloadResult("SELECT X'1234', X'a'")
validateOffloadResult("SELECT DATE'2020-12-31', DATE'2020-12-30'")
- }
+ validateOffloadResult("SELECT CAST(null as struct<u:integer,v:string>)")
+ validateOffloadResult("SELECT array(struct(1, 'a'), null, struct(1, 'a'))")
+ validateOffloadResult(
+ "SELECT array(struct(1, 'a'), " +
+ "CAST(null AS struct<col1:integer,col2:string>))"
+ )
- test("Literal Fallback") {
- validateFallbackResult("SELECT struct(cast(null as struct<a: string>))")
- validateFallbackResult("SELECT array(struct(1, 'a'), null)")
+ validateOffloadResult("SELECT struct(cast(null as struct<a: string>))")
+ validateOffloadResult("SELECT array(null, array('str', 'a'))")
}
}
diff --git a/cpp/velox/substrait/SubstraitToVeloxExpr.cc
b/cpp/velox/substrait/SubstraitToVeloxExpr.cc
index 1fe143597a..518e97fa88 100755
--- a/cpp/velox/substrait/SubstraitToVeloxExpr.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxExpr.cc
@@ -16,6 +16,7 @@
*/
#include "SubstraitToVeloxExpr.h"
+#include <glog/logging.h>
#include "TypeUtils.h"
#include "velox/functions/sparksql/specialforms/SparkCastExpr.h"
#include "velox/type/Timestamp.h"
@@ -72,6 +73,14 @@ ArrayVectorPtr makeEmptyArrayVector(memory::MemoryPool*
pool, const TypePtr& ele
return std::make_shared<ArrayVector>(pool, ARRAY(elementType), nullptr, 1,
offsets, sizes, nullptr);
}
+ArrayVectorPtr makeNullArrayVector(memory::MemoryPool* pool, const TypePtr&
elementType, size_t child_size) {
+ BufferPtr nulls = allocateNulls(1, pool, bits::kNull);
+ BufferPtr offsets = allocateOffsets(1, pool);
+ BufferPtr sizes = allocateOffsets(1, pool);
+
+ return std::make_shared<ArrayVector>(pool, elementType, nulls, child_size,
offsets, sizes, nullptr);
+}
+
MapVectorPtr makeEmptyMapVector(memory::MemoryPool* pool, const TypePtr&
keyType, const TypePtr& valueType) {
BufferPtr offsets = allocateOffsets(1, pool);
BufferPtr sizes = allocateOffsets(1, pool);
@@ -82,6 +91,17 @@ RowVectorPtr makeEmptyRowVector(memory::MemoryPool* pool) {
return makeRowVector({}, {}, 0, pool);
}
+RowVectorPtr makeNullRowVector(memory::MemoryPool* pool, const TypePtr&
rowType) {
+ auto rowTyped = asRowType(rowType);
+ std::vector<VectorPtr> children;
+ children.reserve(rowTyped->size());
+ for (int i = 0; i < rowTyped->size(); i++) {
+ children.push_back(BaseVector::createNullConstant(rowTyped->childAt(i), 1,
pool));
+ }
+ BufferPtr nulls = allocateNulls(1, pool, bits::kNull);
+ return std::make_shared<RowVector>(pool, rowType, nulls, 1, children);
+}
+
template <typename T>
void setLiteralValue(const ::substrait::Expression::Literal& literal,
FlatVector<T>* vector, vector_size_t index) {
if (literal.has_null()) {
@@ -112,6 +132,8 @@ VectorPtr constructFlatVector(
TypePtr getScalarType(const ::substrait::Expression::Literal& literal) {
auto typeCase = literal.literal_type_case();
switch (typeCase) {
+ case ::substrait::Expression_Literal::LiteralTypeCase::kNull:
+ return gluten::SubstraitParser::parseType(literal.null());
case ::substrait::Expression_Literal::LiteralTypeCase::kBoolean:
return BOOLEAN();
case ::substrait::Expression_Literal::LiteralTypeCase::kI8:
@@ -474,7 +496,8 @@ std::shared_ptr<const core::ConstantTypedExpr>
SubstraitVeloxExprConverter::toVe
ArrayVectorPtr SubstraitVeloxExprConverter::literalsToArrayVector(const
::substrait::Expression::Literal& literal) {
auto childSize = literal.list().values().size();
- VELOX_CHECK_GT(childSize, 0, "there should be at least 1 value in list
literal.");
+ auto childType = literal.literal_type_case();
+
auto childLiteral = literal.list().values(0);
auto elementAtFunc = [&](vector_size_t idx) { return
literal.list().values(idx); };
auto childVector = literalsToVector(childLiteral, childSize, elementAtFunc);
@@ -498,37 +521,51 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
vector_size_t childSize,
std::function<::substrait::Expression::Literal(vector_size_t /* idx */)>
elementAtFunc) {
auto childTypeCase = childLiteral.literal_type_case();
- switch (childTypeCase) {
- case ::substrait::Expression_Literal::LiteralTypeCase::kNull: {
- auto veloxType = SubstraitParser::parseType(childLiteral.null());
- auto kind = veloxType->kind();
- return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH_ALL(
- constructFlatVector, kind, elementAtFunc, childSize, veloxType,
pool_);
+ if (childLiteral.has_null()) {
+ auto nullType = SubstraitParser::parseType(childLiteral.null());
+ if (nullType) {
+ switch (nullType->kind()) {
+ case TypeKind::ARRAY:
+ childTypeCase =
::substrait::Expression_Literal::LiteralTypeCase::kList;
+ break;
+ case TypeKind::ROW:
+ childTypeCase =
::substrait::Expression_Literal::LiteralTypeCase::kStruct;
+ break;
+ case TypeKind::MAP:
+ childTypeCase =
::substrait::Expression_Literal::LiteralTypeCase::kMap;
+ break;
+ default:
+ break;
+ }
}
+ }
+ switch (childTypeCase) {
case
::substrait::Expression_Literal::LiteralTypeCase::kIntervalDayToSecond:
return constructFlatVector<TypeKind::BIGINT>(elementAtFunc, childSize,
INTERVAL_DAY_TIME(), pool_);
// Handle EmptyList and List together since the children could be either
case.
case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyList:
case ::substrait::Expression_Literal::LiteralTypeCase::kList: {
- ArrayVectorPtr elements;
+ ArrayVectorPtr arrayVector;
for (int i = 0; i < childSize; i++) {
auto child = elementAtFunc(i);
auto childType = child.literal_type_case();
ArrayVectorPtr grandVector;
-
- if (childType ==
::substrait::Expression_Literal::LiteralTypeCase::kEmptyList) {
+ if (childType ==
::substrait::Expression_Literal::LiteralTypeCase::kNull) {
+ auto veloxType = SubstraitParser::parseType(child.null());
+ grandVector = makeNullArrayVector(pool_, veloxType, 1);
+ } else if (childType ==
::substrait::Expression_Literal::LiteralTypeCase::kEmptyList) {
auto elementType =
SubstraitParser::parseType(child.empty_list().type());
grandVector = makeEmptyArrayVector(pool_, elementType);
} else {
grandVector = literalsToArrayVector(child);
}
- if (!elements) {
- elements = grandVector;
+ if (!arrayVector) {
+ arrayVector = grandVector;
} else {
- elements->append(grandVector.get());
+ arrayVector->append(grandVector.get());
}
}
- return elements;
+ return arrayVector;
}
// Handle EmptyMap and Map together since the children could be either
case.
case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyMap:
@@ -558,7 +595,13 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
RowVectorPtr rowVector;
for (int i = 0; i < childSize; i++) {
auto element = elementAtFunc(i);
- RowVectorPtr grandVector = literalsToRowVector(element);
+ RowVectorPtr grandVector;
+ if (element.literal_type_case() ==
::substrait::Expression_Literal::LiteralTypeCase::kNull) {
+ auto veloxType = SubstraitParser::parseType(element.null());
+ grandVector = makeNullRowVector(pool_, veloxType);
+ } else {
+ grandVector = literalsToRowVector(element);
+ }
if (!rowVector) {
rowVector = grandVector;
} else {
@@ -567,24 +610,37 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
}
return rowVector;
}
- default:
+ default: {
+ // if null, get null type instead of next element type
auto veloxType = getScalarType(elementAtFunc(0));
if (veloxType) {
- auto kind = veloxType->kind();
- return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(
- constructFlatVector, kind, elementAtFunc, childSize, veloxType,
pool_);
+ // Handle UNKNOWN elements (mapped from Spark NullType, as in
array(null))
+ // by creating an all-null vector.
+ if (veloxType->kind() == TypeKind::UNKNOWN) {
+ return BaseVector::createNullConstant(UNKNOWN(), childSize, pool_);
+ }
+ // if veloxType is scalar type
+ if (veloxType->isPrimitiveType()) {
+ auto kind = veloxType->kind();
+ return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(
+ constructFlatVector, kind, elementAtFunc, childSize, veloxType,
pool_);
+ }
}
VELOX_NYI("literals not supported for type case '{}'",
std::to_string(childTypeCase));
+ }
}
}
RowVectorPtr SubstraitVeloxExprConverter::literalsToRowVector(const
::substrait::Expression::Literal& structLiteral) {
- if (structLiteral.has_null()) {
- VELOX_NYI("NULL for struct type is not supported.");
- }
auto numFields = structLiteral.struct_().fields().size();
if (numFields == 0) {
- return makeEmptyRowVector(pool_);
+ auto is_null = structLiteral.literal_type_case() ==
::substrait::Expression_Literal::LiteralTypeCase::kNull;
+ if (!is_null) {
+ return makeEmptyRowVector(pool_);
+ } else {
+ auto veloxType = SubstraitParser::parseType(structLiteral.null());
+ return makeNullRowVector(pool_, veloxType);
+ }
}
std::vector<VectorPtr> vectors;
std::vector<std::string> names;
@@ -603,9 +659,15 @@ RowVectorPtr
SubstraitVeloxExprConverter::literalsToRowVector(const ::substrait:
case ::substrait::Expression_Literal::LiteralTypeCase::kNull: {
auto veloxType = SubstraitParser::parseType(child.null());
auto kind = veloxType->kind();
- auto vecPtr =
- VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(constructFlatVectorForStruct,
kind, child, 1, veloxType, pool_);
- vectors.emplace_back(vecPtr);
+ if (kind == TypeKind::ROW) {
+ vectors.emplace_back(makeNullRowVector(pool_, veloxType));
+ } else if (kind == TypeKind::ARRAY || kind == TypeKind::MAP || kind ==
TypeKind::UNKNOWN) {
+ vectors.emplace_back(BaseVector::createNullConstant(veloxType, 1,
pool_));
+ } else {
+ auto vecPtr =
+ VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(constructFlatVectorForStruct,
kind, child, 1, veloxType, pool_);
+ vectors.emplace_back(vecPtr);
+ }
break;
}
case ::substrait::Expression_Literal::LiteralTypeCase::kList: {
@@ -726,5 +788,4 @@ std::unordered_map<std::string, std::string>
SubstraitVeloxExprConverter::extrac
{"YEAR", "year"},
{"WEEK_OF_YEAR", "week_of_year"},
{"YEAR_OF_WEEK", "year_of_week"}};
-
} // namespace gluten
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c97b6adaa0..f65652d522 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -945,13 +945,21 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for nested struct fields (byPosition)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for nullable array with required element
(byName)")
+ // Overridden.
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for fields inside nullable array (byName)")
+ // Overridden.
.exclude("not null checks for fields inside nullable array (byPosition)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values (byName)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values
(byPosition)")
// Overridden.
+ .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+ // Overridden.
.exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 0ef7eb2da9..fc98d77b98 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,10 +16,12 @@
*/
package org.apache.spark.sql
+import org.apache.gluten.exception.GlutenException
+
import org.apache.spark.SparkException
import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier}
import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
import java.util.Collections
@@ -28,24 +30,129 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
/**
* Shadows Spark's `assertNotNullException`, which is private and so cannot
be reused.
*
- * Spark requires the cause to be a `NullPointerException` and matches the
offending column path
- * against `colPath.mkString("\n", "\n", "\n")`. Velox raises a
`VeloxUserError` that Gluten
- * surfaces as a `SparkException`, carrying the same reason text but not
Spark's column path
- * formatting, so `colPath` is reported on failure rather than asserted. The
reason match is case
- * insensitive because Spark 3.5 words it "Null value appeared ..." and
Spark 4.x "NULL value
- * appeared ...".
+ * Spark expects the cause of the `SparkException` to be a
`NullPointerException` thrown by its
+ * `AssertNotNull`. When the projection is offloaded, Velox
`assert_not_null` raises a
+ * `VeloxUserError` instead, which Gluten surfaces as a `GlutenException`
cause carrying the same
+ * reason text but not Spark's column path formatting, so `colPath` is only
checked when the plan
+ * fell back to vanilla Spark.
*/
private def assertNotNullException(e: SparkException, colPath: Seq[String]):
Unit = {
- val messages = Iterator
- .iterate[Throwable](e)(_.getCause)
- .takeWhile(_ != null)
- .flatMap(t => Option(t.getMessage))
- .mkString("\n")
-
- assert(
- messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in
non-nullable field"),
- s"expected a not-null violation for ${colPath.mkString(".")},
got:\n$messages"
- )
+ e.getCause match {
+ case ge: GlutenException =>
+ assert(
+ ge.getMessage.contains("Null value appeared in non-nullable field"),
+ s"expected a not-null violation for ${colPath.mkString(".")},
got:\n${ge.getMessage}"
+ )
+ case npe: NullPointerException =>
+ assert(npe.getMessage.contains("Null value appeared in non-nullable
field"))
+ assert(npe.getMessage.contains(colPath.mkString("\n", "\n", "\n")))
+ case other =>
+ fail(s"Unexpected exception cause: $other")
+ }
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byName)") {
+ checkNullableArrayWithNotNullElement(byName = true)
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byPosition)") {
+ checkNullableArrayWithNotNullElement(byName = false)
+ }
+
+ private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int").add("y", "int")
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ columns = Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = false))),
+ partitions = Array.empty[Transform],
+ properties = Collections.emptyMap[String, String]
+ )
+
+ if (byName) {
+ val inputDF = sql("SELECT 1 AS i, null AS arr")
+ inputDF.writeTo("t").append()
+ } else {
+ sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+ }
+ checkAnswer(spark.table("t"), Row(1, null))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element"))
+ }
+ }
+
+ testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+ checkNotNullFieldsInsideNullableArray(byName = true)
+ }
+
+ testGluten("not null checks for fields inside nullable array (byPosition)") {
+ checkNotNullFieldsInsideNullableArray(byName = false)
+ }
+
+ private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int", nullable =
false).add("y", "int")
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ columns = Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = true))),
+ partitions = Array.empty[Transform],
+ properties = Collections.emptyMap[String, String]
+ )
+
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element", "x"))
+ }
}
testGluten("NOT NULL checks for nullable map with required values (byName)")
{
@@ -87,7 +194,10 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
}
- /** Only the byPosition case is overridden. */
+ testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+ checkNotNullFieldsInsideNullableMap(byName = true)
+ }
+
testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
checkNotNullFieldsInsideNullableMap(byName = false)
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 50198c58ab..5e754ecd77 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1219,13 +1219,21 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for nested struct fields (byPosition)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for nullable array with required element
(byName)")
+ // Overridden.
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for fields inside nullable array (byName)")
+ // Overridden.
.exclude("not null checks for fields inside nullable array (byPosition)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values (byName)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values
(byPosition)")
// Overridden.
+ .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+ // Overridden.
.exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index d0cd89b643..d8e83175c1 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,10 +16,12 @@
*/
package org.apache.spark.sql
+import org.apache.gluten.exception.GlutenException
+
import org.apache.spark.SparkException
import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier}
import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
import java.util.Collections
@@ -28,23 +30,125 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
/**
* Shadows Spark's `assertNotNullException`, which is private and so cannot
be reused.
*
- * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
- * else, and never reads `colPath`. Velox raises a `VeloxUserError` that
Gluten surfaces as a
- * `SparkException`, which carries the same reason text but not the error
condition, so this
- * asserts the reason and reports `colPath` on failure. The reason match is
case insensitive
- * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL
value appeared ...".
+ * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION` and never reads
+ * `colPath`. When the projection is offloaded, Velox `assert_not_null`
raises a `VeloxUserError`
+ * instead, which reaches the test as a `SparkException` whose cause is a
`GlutenException`
+ * carrying the reason text but not the error condition.
*/
private def assertNotNullException(e: SparkException, colPath: Seq[String]):
Unit = {
- val messages = Iterator
- .iterate[Throwable](e)(_.getCause)
- .takeWhile(_ != null)
- .flatMap(t => Option(t.getMessage))
- .mkString("\n")
-
- assert(
- messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in
non-nullable field"),
- s"expected a not-null violation for ${colPath.mkString(".")},
got:\n$messages"
- )
+ e.getCause match {
+ case ge: GlutenException =>
+ assert(
+ ge.getMessage.contains("Null value appeared in non-nullable field"),
+ s"expected a not-null violation for ${colPath.mkString(".")},
got:\n${ge.getMessage}"
+ )
+ case other =>
+ fail(s"Unexpected exception cause: $other")
+ }
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byName)") {
+ checkNullableArrayWithNotNullElement(byName = true)
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byPosition)") {
+ checkNullableArrayWithNotNullElement(byName = false)
+ }
+
+ private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int").add("y", "int")
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ columns = Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = false))),
+ partitions = Array.empty[Transform],
+ properties = Collections.emptyMap[String, String]
+ )
+
+ if (byName) {
+ val inputDF = sql("SELECT 1 AS i, null AS arr")
+ inputDF.writeTo("t").append()
+ } else {
+ sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+ }
+ checkAnswer(spark.table("t"), Row(1, null))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element"))
+ }
+ }
+
+ testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+ checkNotNullFieldsInsideNullableArray(byName = true)
+ }
+
+ testGluten("not null checks for fields inside nullable array (byPosition)") {
+ checkNotNullFieldsInsideNullableArray(byName = false)
+ }
+
+ private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int", nullable =
false).add("y", "int")
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ columns = Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = true))),
+ partitions = Array.empty[Transform],
+ properties = Collections.emptyMap[String, String]
+ )
+
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element", "x"))
+ }
}
testGluten("NOT NULL checks for nullable map with required values (byName)")
{
@@ -74,7 +178,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, null))
- // Gluten exception differs from Spark
val e = intercept[SparkException] {
if (byName) {
val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -87,7 +190,10 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
}
- /** Only the byPosition case is overridden. */
+ testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+ checkNotNullFieldsInsideNullableMap(byName = true)
+ }
+
testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
checkNotNullFieldsInsideNullableMap(byName = false)
}
@@ -112,7 +218,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
- // Gluten exception differs from Spark
val e1 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
@@ -131,7 +236,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
assertNotNullException(e1, Seq("m", "key", "x"))
- // Gluten exception differs from Spark
val e2 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 1e29452c78..58a2f37ab9 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1221,13 +1221,21 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for nested struct fields (byPosition)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for nullable array with required element
(byName)")
+ // Overridden.
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for fields inside nullable array (byName)")
+ // Overridden.
.exclude("not null checks for fields inside nullable array (byPosition)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values (byName)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values
(byPosition)")
// Overridden.
+ .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+ // Overridden.
.exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 642ad786fb..e677abdce6 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,32 +16,136 @@
*/
package org.apache.spark.sql
+import org.apache.gluten.exception.GlutenException
+
import org.apache.spark.SparkException
import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier,
TableInfo}
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
class GlutenRuntimeNullChecksV2Writes extends RuntimeNullChecksV2Writes with
GlutenSQLTestsTrait {
/**
* Shadows Spark's `assertNotNullException`, which is private and so cannot
be reused.
*
- * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
- * else, and never reads `colPath`. Velox raises a `VeloxUserError` that
Gluten surfaces as a
- * `SparkException`, which carries the same reason text but not the error
condition, so this
- * asserts the reason and reports `colPath` on failure. The reason match is
case insensitive
- * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL
value appeared ...".
+ * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION` and never reads
+ * `colPath`. When the projection is offloaded, Velox `assert_not_null`
raises a `VeloxUserError`
+ * instead, which reaches the test as a `SparkException` whose cause is a
`GlutenException`
+ * carrying the reason text but not the error condition.
*/
private def assertNotNullException(e: SparkException, colPath: Seq[String]):
Unit = {
- val messages = Iterator
- .iterate[Throwable](e)(_.getCause)
- .takeWhile(_ != null)
- .flatMap(t => Option(t.getMessage))
- .mkString("\n")
-
- assert(
- messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in
non-nullable field"),
- s"expected a not-null violation for ${colPath.mkString(".")},
got:\n$messages"
- )
+ e.getCause match {
+ case ge: GlutenException =>
+ assert(
+ ge.getMessage.contains("Null value appeared in non-nullable field"),
+ s"expected a not-null violation for ${colPath.mkString(".")},
got:\n${ge.getMessage}"
+ )
+ case other =>
+ fail(s"Unexpected exception cause: $other")
+ }
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byName)") {
+ checkNullableArrayWithNotNullElement(byName = true)
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byPosition)") {
+ checkNullableArrayWithNotNullElement(byName = false)
+ }
+
+ private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int").add("y", "int")
+ val tableInfo = new TableInfo.Builder()
+ .withColumns(Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = false))))
+ .build()
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ tableInfo = tableInfo)
+
+ if (byName) {
+ val inputDF = sql("SELECT 1 AS i, null AS arr")
+ inputDF.writeTo("t").append()
+ } else {
+ sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+ }
+ checkAnswer(spark.table("t"), Row(1, null))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element"))
+ }
+ }
+
+ testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+ checkNotNullFieldsInsideNullableArray(byName = true)
+ }
+
+ testGluten("not null checks for fields inside nullable array (byPosition)") {
+ checkNotNullFieldsInsideNullableArray(byName = false)
+ }
+
+ private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int", nullable =
false).add("y", "int")
+ val tableInfo = new TableInfo.Builder()
+ .withColumns(Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = true))))
+ .build()
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ tableInfo = tableInfo)
+
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element", "x"))
+ }
}
testGluten("NOT NULL checks for nullable map with required values (byName)")
{
@@ -71,7 +175,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, null))
- // Gluten exception differs from Spark
val e = intercept[SparkException] {
if (byName) {
val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -84,7 +187,10 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
}
- /** Only the byPosition case is overridden. */
+ testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+ checkNotNullFieldsInsideNullableMap(byName = true)
+ }
+
testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
checkNotNullFieldsInsideNullableMap(byName = false)
}
@@ -109,7 +215,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
- // Gluten exception differs from Spark
val e1 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
@@ -128,7 +233,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
assertNotNullException(e1, Seq("m", "key", "x"))
- // Gluten exception differs from Spark
val e2 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 36c9d7c327..cbe607088d 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1315,13 +1315,21 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("NOT NULL checks for nested struct fields (byPosition)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
.exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for nullable array with required element
(byName)")
+ // Overridden.
.exclude("NOT NULL checks for nullable array with required element
(byPosition)")
+ // Overridden.
+ .exclude("NOT NULL checks for fields inside nullable array (byName)")
+ // Overridden.
.exclude("not null checks for fields inside nullable array (byPosition)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values (byName)")
// Overridden.
.exclude("NOT NULL checks for nullable map with required values
(byPosition)")
// Overridden.
+ .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+ // Overridden.
.exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
enableSuite[GlutenTableOptionsConstantFoldingSuite]
enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 642ad786fb..e677abdce6 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,32 +16,136 @@
*/
package org.apache.spark.sql
+import org.apache.gluten.exception.GlutenException
+
import org.apache.spark.SparkException
import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier,
TableInfo}
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
class GlutenRuntimeNullChecksV2Writes extends RuntimeNullChecksV2Writes with
GlutenSQLTestsTrait {
/**
* Shadows Spark's `assertNotNullException`, which is private and so cannot
be reused.
*
- * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
- * else, and never reads `colPath`. Velox raises a `VeloxUserError` that
Gluten surfaces as a
- * `SparkException`, which carries the same reason text but not the error
condition, so this
- * asserts the reason and reports `colPath` on failure. The reason match is
case insensitive
- * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL
value appeared ...".
+ * Spark expects a `SparkRuntimeException` carrying
`NOT_NULL_ASSERT_VIOLATION` and never reads
+ * `colPath`. When the projection is offloaded, Velox `assert_not_null`
raises a `VeloxUserError`
+ * instead, which reaches the test as a `SparkException` whose cause is a
`GlutenException`
+ * carrying the reason text but not the error condition.
*/
private def assertNotNullException(e: SparkException, colPath: Seq[String]):
Unit = {
- val messages = Iterator
- .iterate[Throwable](e)(_.getCause)
- .takeWhile(_ != null)
- .flatMap(t => Option(t.getMessage))
- .mkString("\n")
-
- assert(
- messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in
non-nullable field"),
- s"expected a not-null violation for ${colPath.mkString(".")},
got:\n$messages"
- )
+ e.getCause match {
+ case ge: GlutenException =>
+ assert(
+ ge.getMessage.contains("Null value appeared in non-nullable field"),
+ s"expected a not-null violation for ${colPath.mkString(".")},
got:\n${ge.getMessage}"
+ )
+ case other =>
+ fail(s"Unexpected exception cause: $other")
+ }
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byName)") {
+ checkNullableArrayWithNotNullElement(byName = true)
+ }
+
+ testGluten("NOT NULL checks for nullable array with required element
(byPosition)") {
+ checkNullableArrayWithNotNullElement(byName = false)
+ }
+
+ private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int").add("y", "int")
+ val tableInfo = new TableInfo.Builder()
+ .withColumns(Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = false))))
+ .build()
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ tableInfo = tableInfo)
+
+ if (byName) {
+ val inputDF = sql("SELECT 1 AS i, null AS arr")
+ inputDF.writeTo("t").append()
+ } else {
+ sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+ }
+ checkAnswer(spark.table("t"), Row(1, null))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element"))
+ }
+ }
+
+ testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+ checkNotNullFieldsInsideNullableArray(byName = true)
+ }
+
+ testGluten("not null checks for fields inside nullable array (byPosition)") {
+ checkNotNullFieldsInsideNullableArray(byName = false)
+ }
+
+ private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+ withTable("t") {
+ val structType = new StructType().add("x", "int", nullable =
false).add("y", "int")
+ val tableInfo = new TableInfo.Builder()
+ .withColumns(Array(
+ ColumnV2.create("i", IntegerType),
+ ColumnV2.create("arr", ArrayType(structType, containsNull = true))))
+ .build()
+ catalog.createTable(
+ ident = Identifier.of(Array(), "t"),
+ tableInfo = tableInfo)
+
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+ val e = intercept[SparkException] {
+ if (byName) {
+ val inputDF = sql(
+ s"""SELECT
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr
+ """.stripMargin)
+ inputDF.writeTo("t").append()
+ } else {
+ sql(
+ s"""INSERT INTO t VALUES (
+ | 1 AS i,
+ | array(null, named_struct('x', null, 'y', 1)) AS arr)
+ """.stripMargin)
+ }
+ }
+ assertNotNullException(e, Seq("arr", "element", "x"))
+ }
}
testGluten("NOT NULL checks for nullable map with required values (byName)")
{
@@ -71,7 +175,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, null))
- // Gluten exception differs from Spark
val e = intercept[SparkException] {
if (byName) {
val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -84,7 +187,10 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
}
- /** Only the byPosition case is overridden. */
+ testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+ checkNotNullFieldsInsideNullableMap(byName = true)
+ }
+
testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
checkNotNullFieldsInsideNullableMap(byName = false)
}
@@ -109,7 +215,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
- // Gluten exception differs from Spark
val e1 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
@@ -128,7 +233,6 @@ class GlutenRuntimeNullChecksV2Writes extends
RuntimeNullChecksV2Writes with Glu
}
assertNotNullException(e1, Seq("m", "key", "x"))
- // Gluten exception differs from Spark
val e2 = intercept[SparkException] {
if (byName) {
val inputDF = sql(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]