infvg commented on code in PR #13248:
URL: https://github.com/apache/gluten/pull/13248#discussion_r4246935723
##########
gluten-ut/spark42/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala:
##########
@@ -591,6 +593,186 @@ class GlutenInsertSuite
}
}
}
+
+ // Spark's SPARK-55716 tests expect a
SparkRuntimeException(NOT_NULL_ASSERT_VIOLATION).
+ // Velox assert_not_null throws a VeloxUserError instead, which fails the
query with a
+ // SparkException.
+ private def assertNotNullViolation(f: => Unit): Unit = {
+ val exception = intercept[SparkException](f)
+ val messages = Iterator
+ .iterate[Throwable](exception)(_.getCause)
+ .takeWhile(_ != null)
+ .flatMap(e => Option(e.getMessage))
+
+ assert(messages.exists(_.contains("Null value appeared in non-nullable
field")))
+ }
+
+ testGluten("SPARK-55716: V1 INSERT rejects null into NOT NULL column for
file sources") {
+ Seq("parquet", "orc", "json").foreach {
+ format =>
+ withSQLConf(SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true")
{
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL, s STRING NOT NULL) USING
$format")
+ // V1 DataSource writes now enforce NOT NULL constraints via
AssertNotNull
+ assertNotNullViolation {
+ sql("INSERT INTO t VALUES(null, 'a')")
+ }
+ assertNotNullViolation {
+ sql("INSERT INTO t VALUES(1, null)")
+ }
+ // Valid insert should succeed
+ sql("INSERT INTO t VALUES(1, 'a')")
+ checkAnswer(spark.table("t"), Seq(Row(1, "a")))
+ }
+ }
+ }
+ }
+
+ testGluten("SPARK-55716: V1 INSERT NOT NULL enforcement respects
storeAssignmentPolicy") {
+ Seq("parquet", "orc").foreach {
+ format =>
+ // ANSI mode (default): rejects null
+ withSQLConf(
+ SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true",
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.ANSI.toString) {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL) USING $format")
+ assertNotNullViolation {
+ sql("INSERT INTO t VALUES(null)")
+ }
+ }
+ }
+ // STRICT mode: also rejects null (fails at analysis with type
mismatch)
+ withSQLConf(
+ SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true",
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.STRICT.toString) {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL) USING $format")
+ intercept[AnalysisException] {
+ sql("INSERT INTO t VALUES(null)")
+ }
+ }
+ }
+ // LEGACY mode: allows null (no AssertNotNull injected)
+ withSQLConf(
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.LEGACY.toString) {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL) USING $format")
+ sql("INSERT INTO t VALUES(null)")
+ checkAnswer(spark.table("t"), Seq(Row(null)))
+ }
+ }
+ // Legacy config: allows null even in ANSI mode
+ withSQLConf(SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key ->
"false") {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL) USING $format")
+ sql("INSERT INTO t VALUES(null)")
+ checkAnswer(spark.table("t"), Seq(Row(null)))
+ }
+ }
+ }
+ }
+
+ testGluten("SPARK-55716: V1 INSERT rejects null with V2 file source path") {
+ Seq("parquet", "orc").foreach {
+ format =>
+ withSQLConf(
+ SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true",
+ SQLConf.USE_V1_SOURCE_LIST.key -> "") {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT NOT NULL, s STRING NOT NULL) USING
$format")
+ assertNotNullViolation {
+ sql("INSERT INTO t VALUES(null, 'a')")
+ }
+ }
+ }
+ }
+ }
+
+ testGluten("SPARK-55716: V1 INSERT rejects null array element for NOT NULL
element type") {
+ Seq("parquet", "orc").foreach {
+ format =>
+ withSQLConf(SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true")
{
+ withTable("t") {
+ val schema = new StructType()
+ .add("a", ArrayType(IntegerType, containsNull = false))
+ spark.sessionState.catalog.createTable(
+ CatalogTable(
+ identifier = TableIdentifier("t"),
+ tableType = CatalogTableType.MANAGED,
+ storage = CatalogStorageFormat.empty,
+ schema = schema,
+ provider = Some(format)),
+ ignoreIfExists = false
+ )
+ assertNotNullViolation {
+ sql("INSERT INTO t SELECT array(1, null, 3)")
+ }
+ // Valid insert should succeed
+ sql("INSERT INTO t SELECT array(1, 2, 3)")
+ checkAnswer(spark.table("t"), Seq(Row(Seq(1, 2, 3))))
+ }
+ }
+ }
+ }
+
+ testGluten("SPARK-55716: V1 INSERT rejects null struct field for NOT NULL
field") {
+ Seq("parquet", "orc").foreach {
+ format =>
+ withSQLConf(SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true")
{
+ withTable("t") {
+ val schema = new StructType()
+ .add(
+ "s",
+ new StructType()
+ .add("x", IntegerType, nullable = false)
+ .add("y", StringType, nullable = false))
+ spark.sessionState.catalog.createTable(
+ CatalogTable(
+ identifier = TableIdentifier("t"),
+ tableType = CatalogTableType.MANAGED,
+ storage = CatalogStorageFormat.empty,
+ schema = schema,
+ provider = Some(format)),
+ ignoreIfExists = false
+ )
+ assertNotNullViolation {
+ sql("INSERT INTO t SELECT named_struct('x', null, 'y', 'hello')")
+ }
+ // Valid insert should succeed
+ sql("INSERT INTO t SELECT named_struct('x', 1, 'y', 'hello')")
+ checkAnswer(spark.table("t"), Seq(Row(Row(1, "hello"))))
+ }
+ }
+ }
+ }
+
+ testGluten("SPARK-55716: V1 INSERT rejects null map value for NOT NULL value
type") {
+ Seq("parquet", "orc").foreach {
+ format =>
+ withSQLConf(SQLConf.FILE_SOURCE_INSERT_ENFORCE_NOT_NULL.key -> "true")
{
+ withTable("t") {
+ val schema = new StructType()
+ .add("m", MapType(StringType, IntegerType, valueContainsNull =
false))
+ spark.sessionState.catalog.createTable(
+ CatalogTable(
+ identifier = TableIdentifier("t"),
+ tableType = CatalogTableType.MANAGED,
+ storage = CatalogStorageFormat.empty,
+ schema = schema,
+ provider = Some(format)),
+ ignoreIfExists = false
+ )
+ assertNotNullViolation {
+ sql("INSERT INTO t SELECT map('a', 1, 'b', null)")
+ }
+ // Valid insert should succeed
Review Comment:
These comments are not really necessary
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
