This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2d7179a874 [GLUTEN-11914][VL] Enable Parquet missing struct field
tests for Spark 4.1 and 4.2 (#13398)
2d7179a874 is described below
commit 2d7179a874a7ecbf9fd9c3fd1b6603feb033e5da
Author: rdtr <[email protected]>
AuthorDate: Wed Oct 21 10:13:36 2026 -0700
[GLUTEN-11914][VL] Enable Parquet missing struct field tests for Spark 4.1
and 4.2 (#13398)
---
.../gluten/utils/velox/VeloxTestSettings.scala | 7 +--
.../datasources/parquet/GlutenParquetIOSuite.scala | 56 +++++++++++++++++++++-
.../gluten/utils/velox/VeloxTestSettings.scala | 7 +--
.../datasources/parquet/GlutenParquetIOSuite.scala | 56 +++++++++++++++++++++-
4 files changed, 118 insertions(+), 8 deletions(-)
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index b9110d896c..38af14b69c 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -555,9 +555,10 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-35640: int as long should throw schema incompatible error")
// TODO: fix in Spark-4.0
.exclude("explode nested lists crossing a rowgroup boundary")
- // TODO: fix on Spark-4.1
- .excludeByPrefix("SPARK-53535") // see
https://issues.apache.org/jira/browse/SPARK-53535
- .excludeByPrefix("vectorized reader: missing all struct fields")
+ // These run the scan node alone with VerifyNoAdditionalScanOutputExec,
which does not work
+ // for Gluten's scan node. Rewritten in GlutenParquetIOSuite.
+ .exclude(
+ "SPARK-53535: vectorized reader: missing all struct fields, struct with
complex fields")
.exclude("SPARK-54220: vectorized reader: missing all struct fields,
struct with NullType only")
enableSuite[GlutenParquetV1PartitionDiscoverySuite]
enableSuite[GlutenParquetV2PartitionDiscoverySuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
index 98b6f78db2..4882c573f4 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
@@ -18,7 +18,9 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.spark.sql._
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.sql.types.{ArrayType, BooleanType, IntegerType,
LongType, StringType, StructType}
+
+import scala.jdk.CollectionConverters._
/** A test suite that tests basic Parquet I/O. */
class GlutenParquetIOSuite extends ParquetIOSuite with GlutenSQLTestsBaseTrait
{
@@ -30,6 +32,58 @@ class GlutenParquetIOSuite extends ParquetIOSuite with
GlutenSQLTestsBaseTrait {
spark.read.parquet(testFile(name))
}
+ // The Spark test also runs the scan node alone with
VerifyNoAdditionalScanOutputExec, which
+ // does not work for Gluten's scan node. Check only the result here.
+ testGluten(
+ "SPARK-53535: vectorized reader: missing all struct fields, struct with
complex fields") {
+ val data = Seq(
+ Row(Row(Seq(11, 12, null, 14), Row("21", 22), Row(true)), 100),
+ Row(Row(Seq(11, 12, null, 14), Row("21", 22), Row(false)), 100),
+ Row(null, 100)
+ )
+
+ val tableSchema = new StructType()
+ .add(
+ "_1",
+ new StructType()
+ .add("_1", ArrayType(IntegerType, containsNull = true))
+ .add("_2", new StructType().add("_1", StringType).add("_2",
IntegerType))
+ .add("_3", new StructType().add("_1", BooleanType))
+ )
+ .add("_2", IntegerType)
+
+ val readSchema = new StructType()
+ .add("_1", new StructType().add("_101", IntegerType).add("_102",
LongType))
+
+ withTempPath {
+ path =>
+ val file = path.getCanonicalPath
+ spark.createDataFrame(data.asJava,
tableSchema).write.partitionBy("_2").parquet(file)
+
+ for {
+ offheapEnabled <- Seq(true, false)
+ returnNullStructIfAllFieldsMissing <- Seq(true, false)
+ } {
+ withSQLConf(
+
SQLConf.LEGACY_PARQUET_RETURN_NULL_STRUCT_IF_ALL_FIELDS_MISSING.key ->
+ returnNullStructIfAllFieldsMissing.toString,
+ SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key ->
"true",
+ SQLConf.COLUMN_VECTOR_OFFHEAP_ENABLED.key ->
offheapEnabled.toString
+ ) {
+ val expectedAnswer = if (!returnNullStructIfAllFieldsMissing) {
+ Row(Row(null, null), 100) :: Row(Row(null, null), 100) ::
Row(null, 100) :: Nil
+ } else {
+ Row(null, 100) :: Row(null, 100) :: Row(null, 100) :: Nil
+ }
+
+ withAllParquetReaders {
+ checkAnswer(spark.read.schema(readSchema).parquet(file),
expectedAnswer)
+ }
+ }
+ }
+ }
+ }
+
testGluten(
"SPARK-54220: vectorized reader: missing all struct fields, struct with
NullType only") {
val data = Seq(
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index ac13d5f467..ee3ee1aee1 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -593,9 +593,10 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-35640: int as long should throw schema incompatible error")
// TODO: fix in Spark-4.0
.exclude("explode nested lists crossing a rowgroup boundary")
- // TODO: fix on Spark-4.1
- .excludeByPrefix("SPARK-53535") // see
https://issues.apache.org/jira/browse/SPARK-53535
- .excludeByPrefix("vectorized reader: missing all struct fields")
+ // These run the scan node alone with VerifyNoAdditionalScanOutputExec,
which does not work
+ // for Gluten's scan node. Rewritten in GlutenParquetIOSuite.
+ .exclude(
+ "SPARK-53535: vectorized reader: missing all struct fields, struct with
complex fields")
.exclude("SPARK-54220: vectorized reader: missing all struct fields,
struct with NullType only")
.exclude("SPARK-54220: NullType") // GLUTEN-12569 Spark 4.2 UT enablement
(CI-confirmed)
enableSuite[GlutenParquetV1PartitionDiscoverySuite]
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
index 98b6f78db2..4882c573f4 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
@@ -18,7 +18,9 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.spark.sql._
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
+import org.apache.spark.sql.types.{ArrayType, BooleanType, IntegerType,
LongType, StringType, StructType}
+
+import scala.jdk.CollectionConverters._
/** A test suite that tests basic Parquet I/O. */
class GlutenParquetIOSuite extends ParquetIOSuite with GlutenSQLTestsBaseTrait
{
@@ -30,6 +32,58 @@ class GlutenParquetIOSuite extends ParquetIOSuite with
GlutenSQLTestsBaseTrait {
spark.read.parquet(testFile(name))
}
+ // The Spark test also runs the scan node alone with
VerifyNoAdditionalScanOutputExec, which
+ // does not work for Gluten's scan node. Check only the result here.
+ testGluten(
+ "SPARK-53535: vectorized reader: missing all struct fields, struct with
complex fields") {
+ val data = Seq(
+ Row(Row(Seq(11, 12, null, 14), Row("21", 22), Row(true)), 100),
+ Row(Row(Seq(11, 12, null, 14), Row("21", 22), Row(false)), 100),
+ Row(null, 100)
+ )
+
+ val tableSchema = new StructType()
+ .add(
+ "_1",
+ new StructType()
+ .add("_1", ArrayType(IntegerType, containsNull = true))
+ .add("_2", new StructType().add("_1", StringType).add("_2",
IntegerType))
+ .add("_3", new StructType().add("_1", BooleanType))
+ )
+ .add("_2", IntegerType)
+
+ val readSchema = new StructType()
+ .add("_1", new StructType().add("_101", IntegerType).add("_102",
LongType))
+
+ withTempPath {
+ path =>
+ val file = path.getCanonicalPath
+ spark.createDataFrame(data.asJava,
tableSchema).write.partitionBy("_2").parquet(file)
+
+ for {
+ offheapEnabled <- Seq(true, false)
+ returnNullStructIfAllFieldsMissing <- Seq(true, false)
+ } {
+ withSQLConf(
+
SQLConf.LEGACY_PARQUET_RETURN_NULL_STRUCT_IF_ALL_FIELDS_MISSING.key ->
+ returnNullStructIfAllFieldsMissing.toString,
+ SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key ->
"true",
+ SQLConf.COLUMN_VECTOR_OFFHEAP_ENABLED.key ->
offheapEnabled.toString
+ ) {
+ val expectedAnswer = if (!returnNullStructIfAllFieldsMissing) {
+ Row(Row(null, null), 100) :: Row(Row(null, null), 100) ::
Row(null, 100) :: Nil
+ } else {
+ Row(null, 100) :: Row(null, 100) :: Row(null, 100) :: Nil
+ }
+
+ withAllParquetReaders {
+ checkAnswer(spark.read.schema(readSchema).parquet(file),
expectedAnswer)
+ }
+ }
+ }
+ }
+ }
+
testGluten(
"SPARK-54220: vectorized reader: missing all struct fields, struct with
NullType only") {
val data = Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
