This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3195aa8c8d [VL] Enable native Parquet write for complex types
(Struct/Array/Map) (#11788)
3195aa8c8d is described below
commit 3195aa8c8dc2f88701d8c935b152d435a9287aa6
Author: Zouxxyy <[email protected]>
AuthorDate: Mon Mar 30 17:29:29 2026 +0800
[VL] Enable native Parquet write for complex types (Struct/Array/Map)
(#11788)
---
.../spark/sql/delta/GlutenParquetFileFormat.scala | 52 +++------
.../delta/files/GlutenDeltaFileFormatWriter.scala | 4 +-
.../gluten/backendsapi/velox/VeloxBackend.scala | 128 ++++++++-------------
.../execution/VeloxParquetWriteForHiveSuite.scala | 36 ++++++
.../sql/execution/VeloxParquetWriteSuite.scala | 55 ++++++++-
.../gluten/backendsapi/BackendSettingsApi.scala | 2 -
.../datasources/GlutenWriterColumnarRules.scala | 5 +-
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +
9 files changed, 166 insertions(+), 124 deletions(-)
diff --git
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/GlutenParquetFileFormat.scala
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/GlutenParquetFileFormat.scala
index 91bc39bd2e..038afa3898 100644
---
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/GlutenParquetFileFormat.scala
+++
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/GlutenParquetFileFormat.scala
@@ -16,7 +16,6 @@
*/
package org.apache.spark.sql.delta
-import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.execution.datasource.GlutenFormatFactory
import org.apache.spark.internal.Logging
@@ -31,16 +30,11 @@ import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
import org.apache.parquet.hadoop.ParquetOutputFormat
import org.apache.parquet.hadoop.codec.CodecConfig
import org.apache.parquet.hadoop.util.ContextUtil
-import org.slf4j.LoggerFactory
-
class GlutenParquetFileFormat
extends ParquetFileFormat
with DataSourceRegister
with Logging
with Serializable {
- import GlutenParquetFileFormat._
-
- private val logger =
LoggerFactory.getLogger(classOf[GlutenParquetFileFormat])
override def shortName(): String = "gluten-parquet"
@@ -62,39 +56,27 @@ class GlutenParquetFileFormat
job: Job,
options: Map[String, String],
dataSchema: StructType): OutputWriterFactory = {
- if (isNativeWritable(dataSchema)) {
- // Pass compression to job conf so that the file extension can be aware
of it.
- val conf = ContextUtil.getConfiguration(job)
- val parquetOptions = new ParquetOptions(options,
sparkSession.sessionState.conf)
- conf.set(ParquetOutputFormat.COMPRESSION,
parquetOptions.compressionCodecClassName)
- val nativeConf =
- GlutenFormatFactory("parquet")
- .nativeConf(options, parquetOptions.compressionCodecClassName)
+ // Pass compression to job conf so that the file extension can be aware of
it.
+ val conf = ContextUtil.getConfiguration(job)
+ val parquetOptions = new ParquetOptions(options,
sparkSession.sessionState.conf)
+ conf.set(ParquetOutputFormat.COMPRESSION,
parquetOptions.compressionCodecClassName)
+ val nativeConf =
+ GlutenFormatFactory("parquet")
+ .nativeConf(options, parquetOptions.compressionCodecClassName)
- return new OutputWriterFactory {
- override def getFileExtension(context: TaskAttemptContext): String = {
- CodecConfig.from(context).getCodec.getExtension + ".parquet"
- }
+ new OutputWriterFactory {
+ override def getFileExtension(context: TaskAttemptContext): String = {
+ CodecConfig.from(context).getCodec.getExtension + ".parquet"
+ }
- override def newInstance(
- path: String,
- dataSchema: StructType,
- context: TaskAttemptContext): OutputWriter = {
- GlutenFormatFactory("parquet")
- .createOutputWriter(path, dataSchema, context, nativeConf)
+ override def newInstance(
+ path: String,
+ dataSchema: StructType,
+ context: TaskAttemptContext): OutputWriter = {
+ GlutenFormatFactory("parquet")
+ .createOutputWriter(path, dataSchema, context, nativeConf)
- }
}
}
- logger.warn(
- s"Data schema is unsupported by Gluten Parquet writer: $dataSchema, " +
- s"falling back to the vanilla Spark Parquet writer")
- super.prepareWrite(sparkSession, job, options, dataSchema)
- }
-}
-
-object GlutenParquetFileFormat {
- def isNativeWritable(schema: StructType): Boolean = {
- BackendsApiManager.getSettings.supportNativeWrite(schema.fields)
}
}
diff --git
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/files/GlutenDeltaFileFormatWriter.scala
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/files/GlutenDeltaFileFormatWriter.scala
index 653b705bec..ee3b49e47e 100644
---
a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/files/GlutenDeltaFileFormatWriter.scala
+++
b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/files/GlutenDeltaFileFormatWriter.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
import org.apache.spark.sql.connector.write.WriterCommitMessage
-import org.apache.spark.sql.delta.{DeltaOptions, GlutenParquetFileFormat}
+import org.apache.spark.sql.delta.DeltaOptions
import org.apache.spark.sql.delta.logging.DeltaLogKeys
import org.apache.spark.sql.delta.stats.GlutenDeltaJobStatsTracker
import org.apache.spark.sql.errors.QueryExecutionErrors
@@ -126,7 +126,7 @@ object GlutenDeltaFileFormatWriter extends LoggingShims {
val dataSchema = dataColumns.toStructType
DataSourceUtils.verifySchema(fileFormat, dataSchema)
DataSourceUtils.checkFieldNames(fileFormat, dataSchema)
- val isNativeWritable = GlutenParquetFileFormat.isNativeWritable(dataSchema)
+ val isNativeWritable = true
val outputDataColumns =
if
(caseInsensitiveOptions.get(DeltaOptions.WRITE_PARTITION_COLUMNS).contains("true"))
{
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index 9694c8a3ae..1ac9f264a2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -284,27 +284,25 @@ object VeloxBackendSettings extends BackendSettingsApi {
isPartitionedTable: Boolean,
options: Map[String, String]): ValidationResult = {
- // Validate if HiveFileFormat write is supported based on output file type
- def validateHiveFileFormat(hiveFileFormat: HiveFileFormat): Option[String]
= {
- // Reflect to get access to fileSinkConf which contains the output file
format
- val fileSinkConfField = format.getClass.getDeclaredField("fileSinkConf")
- fileSinkConfField.setAccessible(true)
- val fileSinkConf = fileSinkConfField.get(hiveFileFormat)
- val tableInfoField = fileSinkConf.getClass.getDeclaredField("tableInfo")
- tableInfoField.setAccessible(true)
- val tableInfo = tableInfoField.get(fileSinkConf)
- val getOutputFileFormatClassNameMethod = tableInfo.getClass
- .getDeclaredMethod("getOutputFileFormatClassName")
- val outputFileFormatClassName =
getOutputFileFormatClassNameMethod.invoke(tableInfo)
-
- // Match based on the output file format class name
- outputFileFormatClassName match {
- case "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
=>
- None
- case _ =>
- Some(
- "HiveFileFormat is supported only with Parquet as the output file
type"
- ) // Unsupported format
+ def validateFileFormat(): Option[String] = {
+ format match {
+ case _: ParquetFileFormat => None
+ case h: HiveFileFormat if GlutenConfig.get.enableHiveFileFormatWriter
=>
+ // Validate HiveFileFormat is backed by Parquet
+ val fileSinkConfField =
format.getClass.getDeclaredField("fileSinkConf")
+ fileSinkConfField.setAccessible(true)
+ val fileSinkConf = fileSinkConfField.get(h)
+ val tableInfoField =
fileSinkConf.getClass.getDeclaredField("tableInfo")
+ tableInfoField.setAccessible(true)
+ val tableInfo = tableInfoField.get(fileSinkConf)
+ val outputFormat = tableInfo.getClass
+ .getDeclaredMethod("getOutputFileFormatClassName")
+ .invoke(tableInfo)
+ outputFormat match {
+ case
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" => None
+ case _ => Some("HiveFileFormat is supported only with Parquet as
the output file type")
+ }
+ case _ => Some("Only ParquetFileFormat and HiveFileFormat are
supported.")
}
}
@@ -318,55 +316,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}
- // Validate if all types are supported.
- def validateDataTypes(): Option[String] = {
- val unsupportedTypes = format match {
- case _: ParquetFileFormat =>
- fields.flatMap {
- case StructField(_, _: YearMonthIntervalType, _, _) =>
- Some("YearMonthIntervalType")
- case StructField(_, _: StructType, _, _) =>
- Some("StructType")
- case _ => None
- }
- case _ =>
- fields.flatMap {
- field =>
- field.dataType match {
- case _: StructType => Some("StructType")
- case _: ArrayType => Some("ArrayType")
- case _: MapType => Some("MapType")
- case _: YearMonthIntervalType => Some("YearMonthIntervalType")
- case _ => None
- }
- }
- }
- if (unsupportedTypes.nonEmpty) {
- Some(unsupportedTypes.mkString("Found unsupported type:", ",", ""))
- } else {
- None
- }
- }
-
- def validateFieldMetadata(): Option[String] = {
- fields.find(_.metadata != Metadata.empty).map {
- filed =>
- s"StructField contain the metadata information: $filed, metadata:
${filed.metadata}"
- }
- }
-
- def validateFileFormat(): Option[String] = {
- format match {
- case _: ParquetFileFormat => None // Parquet is directly supported
- case h: HiveFileFormat if GlutenConfig.get.enableHiveFileFormatWriter
=>
- validateHiveFileFormat(h) // Parquet via Hive SerDe
- case _ =>
- Some(
- "Only ParquetFileFormat and HiveFileFormat are supported."
- ) // Unsupported format
- }
- }
-
def validateWriteFilesOptions(): Option[String] = {
val maxRecordsPerFile = options
.get("maxRecordsPerFile")
@@ -394,8 +343,35 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}
- validateCompressionCodec()
- .orElse(validateFileFormat())
+ def validateDataTypes(): Option[String] = {
+ def hasUnsupportedType(dt: DataType): Boolean = dt match {
+ case _: YearMonthIntervalType => true
+ case st: StructType => st.fields.exists(f =>
hasUnsupportedType(f.dataType))
+ case at: ArrayType => hasUnsupportedType(at.elementType)
+ case mt: MapType => hasUnsupportedType(mt.keyType) ||
hasUnsupportedType(mt.valueType)
+ case _ => false
+ }
+
+ val unsupported = fields.filter(f => hasUnsupportedType(f.dataType))
+ if (unsupported.nonEmpty) {
+ Some(
+ unsupported
+ .map(_.dataType.simpleString)
+ .mkString("Found unsupported type:", ",", ""))
+ } else {
+ None
+ }
+ }
+
+ def validateFieldMetadata(): Option[String] = {
+ fields.find(_.metadata != Metadata.empty).map {
+ filed =>
+ s"StructField contain the metadata information: $filed, metadata:
${filed.metadata}"
+ }
+ }
+
+ validateFileFormat()
+ .orElse(validateCompressionCodec())
.orElse(validateFieldMetadata())
.orElse(validateDataTypes())
.orElse(validateWriteFilesOptions())
@@ -405,14 +381,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}
- override def supportNativeWrite(fields: Array[StructField]): Boolean = {
- def isNotSupported(dataType: DataType): Boolean = dataType match {
- case _: StructType | _: ArrayType | _: MapType => true
- case _ => false
- }
- !fields.exists(field => isNotSupported(field.dataType))
- }
-
override def supportExpandExec(): Boolean = true
override def supportSortExec(): Boolean = true
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
index 2522e2898d..cdd36201ec 100644
---
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
@@ -199,6 +199,42 @@ class VeloxParquetWriteForHiveSuite extends
GlutenQueryTest with SQLTestUtils wi
}
}
+ test("test hive write with struct type") {
+ withTable("t", "src") {
+ withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") {
+ spark.sql("CREATE TABLE src (info STRUCT<name: STRING, age: INT>)
STORED AS PARQUET")
+ spark.sql("INSERT INTO src SELECT named_struct('name', 'alice', 'age',
30)")
+ spark.sql("CREATE TABLE t (info STRUCT<name: STRING, age: INT>) STORED
AS PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT info FROM src")
+ checkAnswer(spark.table("t"), Row(Row("alice", 30)))
+ }
+ }
+ }
+
+ test("test hive write with array type") {
+ withTable("t", "src") {
+ withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") {
+ spark.sql("CREATE TABLE src (ids ARRAY<INT>) STORED AS PARQUET")
+ spark.sql("INSERT INTO src SELECT array(1, 2, 3)")
+ spark.sql("CREATE TABLE t (ids ARRAY<INT>) STORED AS PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT ids FROM src")
+ checkAnswer(spark.table("t"), Row(Seq(1, 2, 3)))
+ }
+ }
+ }
+
+ test("test hive write with map type") {
+ withTable("t", "src") {
+ withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") {
+ spark.sql("CREATE TABLE src (kv MAP<STRING, INT>) STORED AS PARQUET")
+ spark.sql("INSERT INTO src SELECT map('a', 1, 'b', 2)")
+ spark.sql("CREATE TABLE t (kv MAP<STRING, INT>) STORED AS PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT kv FROM src")
+ checkAnswer(spark.table("t"), Row(Map("a" -> 1, "b" -> 2)))
+ }
+ }
+ }
+
test("test hive write dir") {
withTempPath {
f =>
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala
index 1b0d0647dd..a4577c8a5b 100644
---
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala
@@ -28,6 +28,7 @@ import org.apache.parquet.hadoop.ParquetFileReader
import org.apache.parquet.hadoop.util.HadoopInputFile
class VeloxParquetWriteSuite extends VeloxWholeStageTransformerSuite with
WriteUtils {
+
override protected val resourcePath: String = "/tpch-data-parquet"
override protected val fileFormat: String = "parquet"
@@ -62,9 +63,10 @@ class VeloxParquetWriteSuite extends
VeloxWholeStageTransformerSuite with WriteU
withTempPath {
f =>
val path = f.getCanonicalPath
+ // TODO: maybe remove constant complex type restriction (Spark 3.4+)
checkNativeWrite(
s"INSERT OVERWRITE DIRECTORY '$path' USING PARQUET SELECT
array(struct(1), null) as var1",
- expectNative = false)
+ expectNative = !isSparkVersionGE("3.4"))
}
}
@@ -120,6 +122,57 @@ class VeloxParquetWriteSuite extends
VeloxWholeStageTransformerSuite with WriteU
}
}
+ test("test insert into with struct type") {
+ withTable("t", "src") {
+ spark.sql("CREATE TABLE src (info STRUCT<name: STRING, age: INT>) USING
PARQUET")
+ checkNativeWrite("INSERT INTO src SELECT named_struct('name', 'alice',
'age', 30)")
+ spark.sql("CREATE TABLE t (info STRUCT<name: STRING, age: INT>) USING
PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT info FROM src")
+ checkAnswer(spark.table("t"), Row(Row("alice", 30)))
+ }
+ }
+
+ test("test insert into with array type") {
+ withTable("t", "src") {
+ spark.sql("CREATE TABLE src (ids ARRAY<INT>) USING PARQUET")
+ // TODO: maybe remove constant complex type restriction (Spark 3.4+)
+ checkNativeWrite(
+ "INSERT INTO src SELECT array(1, 2, 3)",
+ expectNative = !isSparkVersionGE("3.4"))
+ spark.sql("CREATE TABLE t (ids ARRAY<INT>) USING PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT ids FROM src")
+ checkAnswer(spark.table("t"), Row(Seq(1, 2, 3)))
+ }
+ }
+
+ test("test insert into with map type") {
+ withTable("t", "src") {
+ spark.sql("CREATE TABLE src (kv MAP<STRING, INT>) USING PARQUET")
+ // TODO: maybe remove constant complex type restriction (Spark 3.4+)
+ checkNativeWrite(
+ "INSERT INTO src SELECT map('a', 1, 'b', 2)",
+ expectNative = !isSparkVersionGE("3.4"))
+ spark.sql("CREATE TABLE t (kv MAP<STRING, INT>) USING PARQUET")
+ checkNativeWrite("INSERT INTO t SELECT kv FROM src")
+ checkAnswer(spark.table("t"), Row(Map("a" -> 1, "b" -> 2)))
+ }
+ }
+
+ test("test insert into with nested struct type") {
+ withTable("t", "src") {
+ spark.sql(
+ """CREATE TABLE src (info STRUCT<name: STRING, addr: STRUCT<city:
STRING, zip: INT>>)
+ |USING PARQUET""".stripMargin)
+ checkNativeWrite(
+ "INSERT INTO src SELECT named_struct('name', 'alice', " +
+ "'addr', named_struct('city', 'hangzhou', 'zip', 310000))")
+ spark.sql("""CREATE TABLE t (info STRUCT<name: STRING, addr:
STRUCT<city: STRING, zip: INT>>)
+ |USING PARQUET""".stripMargin)
+ checkNativeWrite("INSERT INTO t SELECT info FROM src")
+ checkAnswer(spark.table("t"), Row(Row("alice", Row("hangzhou", 310000))))
+ }
+ }
+
test("test ctas") {
withTable("velox_ctas") {
spark
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index 8dd3156099..79274b4c60 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -59,8 +59,6 @@ trait BackendSettingsApi {
isPartitionedTable: Boolean,
options: Map[String, String]): ValidationResult =
ValidationResult.succeeded
- def supportNativeWrite(fields: Array[StructField]): Boolean = true
-
def supportNativeMetadataColumns(): Boolean = true
def supportNativeRowIndexColumn(): Boolean = true
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
index 606fa377b8..346d6b7f4d 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala
@@ -101,10 +101,7 @@ object GlutenWriterColumnarRules {
case rc @ DataWritingCommandExec(cmd, child) =>
// The same thread can set these properties in the last query
submission.
val format =
- if (
-
BackendsApiManager.getSettings.supportNativeWrite(child.schema.fields) &&
- BackendsApiManager.getSettings.enableNativeWriteFiles()
- ) {
+ if (BackendsApiManager.getSettings.enableNativeWriteFiles()) {
getNativeFormat(cmd)
} else {
None
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 160fcae2a9..3c8af7bff6 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -864,6 +864,10 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenVariantEndToEndSuite]
enableSuite[GlutenVariantShreddingSuite]
enableSuite[GlutenVariantSuite]
+ // TODO: Velox parquet writer marks all struct fields as OPTIONAL
(nullable),
+ // but Spark's variant type requires REQUIRED fields. Needs Velox-side
fix.
+ .exclude("SPARK-47546: invalid variant binary")
+ .exclude("SPARK-47546: valid variant binary")
enableSuite[GlutenVariantWriteShreddingSuite]
enableSuite[GlutenXmlFunctionsSuite]
enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 45539b768a..dc149341ab 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -835,6 +835,10 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenVariantEndToEndSuite]
enableSuite[GlutenVariantShreddingSuite]
enableSuite[GlutenVariantSuite]
+ // TODO: Velox parquet writer marks all struct fields as OPTIONAL
(nullable),
+ // but Spark's variant type requires REQUIRED fields. Needs Velox-side
fix.
+ .exclude("SPARK-47546: invalid variant binary")
+ .exclude("SPARK-47546: valid variant binary")
enableSuite[GlutenVariantWriteShreddingSuite]
enableSuite[GlutenXmlFunctionsSuite]
enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
