This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6d6a1afae3 [GLUTEN-1433][VL] Fix TimestampNTZ schema validation
(#12029)
6d6a1afae3 is described below
commit 6d6a1afae359a3b033d747c8d56dc2fc45481da2
Author: Joey <[email protected]>
AuthorDate: Thu May 7 07:50:29 2026 +0800
[GLUTEN-1433][VL] Fix TimestampNTZ schema validation (#12029)
---
.../gluten/backendsapi/velox/VeloxValidatorApi.scala | 7 +------
.../execution/VeloxParquetDataTypeValidationSuite.scala | 16 +++++++++++++++-
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
index 9ff33229fa..f46b8b0354 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
@@ -111,12 +111,7 @@ object VeloxValidatorApi {
StringType | BinaryType | _: DecimalType | DateType | TimestampType |
YearMonthIntervalType.DEFAULT | NullType =>
true
- case dt
- if !enableTimestampNtzValidation &&
- dt.getClass.getSimpleName == "TimestampNTZType" =>
- // Allow TimestampNTZ when validation is disabled (for
development/testing)
- // Use reflection to avoid compile-time dependency on Spark 3.4+
TimestampNTZType
- true
+ case dt if !enableTimestampNtzValidation && dt.catalogString ==
"timestamp_ntz" => true
case _ => false
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
index a5e814dc0f..9e46f2a2a7 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
@@ -16,7 +16,8 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.backendsapi.velox.VeloxValidatorApi
+import org.apache.gluten.config.{GlutenConfig, VeloxConfig}
import org.apache.spark.SparkConf
@@ -479,6 +480,19 @@ class VeloxParquetDataTypeValidationSuite extends
VeloxWholeStageTransformerSuit
}
}
+ testWithMinSparkVersion(
+ "Schema validation for TimestampNTZ respects enableTimestampNtzValidation",
+ "3.4") {
+ val ntzType = spark.sql("SELECT
TIMESTAMP_NTZ'2024-01-01'").schema.head.dataType
+ Seq("true", "false").foreach {
+ enabled =>
+ withSQLConf(VeloxConfig.ENABLE_TIMESTAMP_NTZ_VALIDATION.key ->
enabled) {
+ val result = VeloxValidatorApi.validateSchema(ntzType)
+ assert(result.isDefined == enabled.toBoolean)
+ }
+ }
+ }
+
test("Velox Parquet Write") {
withSQLConf((GlutenConfig.NATIVE_WRITER_ENABLED.key, "true")) {
withTempDir {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
