This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6a88b0a8c3 [GLUTEN-11921] Enable Parquet read/write test for NullType
(#12056)
6a88b0a8c3 is described below
commit 6a88b0a8c3e39fa4a776a57554b2b1a45cfebc43
Author: Wechar Yu <[email protected]>
AuthorDate: Wed Sep 2 02:24:10 2026 +0800
[GLUTEN-11921] Enable Parquet read/write test for NullType (#12056)
---
cpp/core/config/GlutenConfig.h | 3 ++
cpp/velox/utils/ConfigExtractor.cc | 2 +-
.../org/apache/gluten/config/GlutenConfig.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 +-
.../datasources/parquet/GlutenParquetIOSuite.scala | 34 ++++++++++++++++++++++
5 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index dae818c57c..9d4f12e871 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -76,6 +76,9 @@ const std::string kParquetWriterVersion =
"parquet.writer.version";
const std::string kParquetCompressionCodec =
"spark.sql.parquet.compression.codec";
+const std::string kLegacyParquetReturnNullStructIfAllFieldsMissing =
+ "spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing";
+
/// Spark `spark.sql.parquet.writeLegacyFormat` (passed from the JVM as
"true"/"false").
/// Used in VeloxWriterUtils to set
`WriterOptions::enableStoreDecimalAsInteger` (inverted).
/// Velox decimal storage when enableStoreDecimalAsInteger is:
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index ed2850a811..544202886e 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -276,7 +276,7 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
createHiveConnectorSessionC
configs[parquetSessionProperty(facebook::velox::parquet::ParquetConfig::kWriterDictionaryPageSizeLimitSession)]
=
conf->get<std::string>(kWriteParquetDictSizeBytes, "2MB");
configs[parquetSessionProperty(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)]
=
- "true";
+ conf->get<bool>(kLegacyParquetReturnNullStructIfAllFieldsMissing, true)
? "true" : "false";
overwriteVeloxConf(conf.get(), configs, kDynamicBackendConfPrefix);
return
std::make_shared<facebook::velox::config::ConfigBase>(std::move(configs));
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 39d5acfb33..575c111d6c 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -505,6 +505,7 @@ object GlutenConfig extends ConfigRegistry {
SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS.key,
"spark.io.compression.codec",
"spark.sql.decimalOperations.allowPrecisionLoss",
+ "spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing",
// s3 config
SPARK_S3_ACCESS_KEY,
SPARK_S3_SECRET_KEY,
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 199068737c..61f202afdd 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -567,7 +567,7 @@ class VeloxTestSettings extends BackendTestSettings {
// TODO: fix on Spark-4.1
.excludeByPrefix("SPARK-53535") // see
https://issues.apache.org/jira/browse/SPARK-53535
.excludeByPrefix("vectorized reader: missing all struct fields")
- .excludeByPrefix("SPARK-54220") //
https://issues.apache.org/jira/browse/SPARK-54220
+ .exclude("SPARK-54220: vectorized reader: missing all struct fields,
struct with NullType only")
enableSuite[GlutenParquetV1PartitionDiscoverySuite]
enableSuite[GlutenParquetV2PartitionDiscoverySuite]
enableSuite[GlutenParquetProtobufCompatibilitySuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
index ad1ae40f92..98b6f78db2 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution.datasources.parquet
import org.apache.spark.sql._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
/** A test suite that tests basic Parquet I/O. */
class GlutenParquetIOSuite extends ParquetIOSuite with GlutenSQLTestsBaseTrait
{
@@ -27,4 +29,36 @@ class GlutenParquetIOSuite extends ParquetIOSuite with
GlutenSQLTestsBaseTrait {
override protected def readResourceParquetFile(name: String): DataFrame = {
spark.read.parquet(testFile(name))
}
+
+ testGluten(
+ "SPARK-54220: vectorized reader: missing all struct fields, struct with
NullType only") {
+ val data = Seq(
+ Tuple1((null, null)),
+ Tuple1((null, null)),
+ Tuple1(null)
+ )
+ val readSchema = new StructType().add(
+ "_1",
+ new StructType()
+ .add("_3", IntegerType, nullable = true)
+ .add("_4", StringType, nullable = true),
+ nullable = true)
+ val expectedAnswer = Row(Row(null, null)) :: Row(Row(null, null)) ::
Row(null) :: Nil
+
+ withParquetFile(data) {
+ file =>
+ for (offheapEnabled <- Seq(true, false)) {
+ withSQLConf(
+ SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key ->
"true",
+
SQLConf.LEGACY_PARQUET_RETURN_NULL_STRUCT_IF_ALL_FIELDS_MISSING.key -> "false",
+ SQLConf.COLUMN_VECTOR_OFFHEAP_ENABLED.key ->
offheapEnabled.toString
+ ) {
+ withAllParquetReaders {
+ val df = spark.read.schema(readSchema).parquet(file)
+ checkAnswer(df, expectedAnswer)
+ }
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
