This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3ae572bc27 [GLUTEN-11622][VL] Support cast between TIMESTAMP and
TIMESTAMP_NTZ (#12654)
3ae572bc27 is described below
commit 3ae572bc2740d028538f549f437557b34f734480
Author: Mariam AlMesfer <[email protected]>
AuthorDate: Fri Aug 14 12:54:27 2026 +0300
[GLUTEN-11622][VL] Support cast between TIMESTAMP and TIMESTAMP_NTZ (#12654)
Co-authored-by: Mariam-Almesfer <[email protected]>
---
.../functions/DateFunctionsValidateSuite.scala | 46 ++++++++++++++++++++++
.../extension/columnar/validator/Validators.scala | 7 +++-
2 files changed, 51 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 36ce82257a..95619f0e0b 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -615,6 +615,52 @@ class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
runQueryAndCompare("select timestampadd(hour, 1, ts) from view") {
checkGlutenPlan[ProjectExecTransformer]
}
+
+ // cast(timestamp_ntz as timestamp)
+ runQueryAndCompare("select cast(ts as timestamp) from view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+
+ withSQLConf("spark.sql.session.timeZone" -> "Asia/Hong_Kong") {
+ val dstPath = dir.getAbsolutePath + "/dst_gap"
+ spark
+ .createDataset(Seq("1941-12-25 00:00:00"))
+ .toDF("input")
+ .selectExpr("cast(input as timestamp_ntz) as ts")
+ .coalesce(1)
+ .write
+ .mode("overwrite")
+ .parquet(dstPath)
+ spark.read.parquet(dstPath).createOrReplaceTempView("dst_gap_view")
+ runQueryAndCompare("select cast(ts as timestamp) from dst_gap_view")
{
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+
+ withTempPath {
+ dir =>
+ val path = dir.getAbsolutePath
+ spark
+ .createDataset(inputs)
+ .toDF("input")
+ .selectExpr("cast(input as timestamp) as ts")
+ .coalesce(1)
+ .write
+ .mode("overwrite")
+ .parquet(path)
+ spark.read.parquet(path).createOrReplaceTempView("ts_view")
+
+ // cast(timestamp as timestamp_ntz)
+ runQueryAndCompare("select cast(ts as timestamp_ntz) from ts_view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+
+ withSQLConf("spark.sql.session.timeZone" -> "UTC") {
+ runQueryAndCompare("select cast(ts as timestamp_ntz) from ts_view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
}
}
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
index f3825ed89a..b7be523c6d 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
@@ -25,7 +25,7 @@ import
org.apache.gluten.extension.columnar.offload.OffloadSingleNode
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.expressions.{Hour, Minute, Second,
TimestampAdd}
+import org.apache.spark.sql.catalyst.expressions.{Cast, Hour, Minute, Second,
TimestampAdd}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.aggregate.{HashAggregateExec,
ObjectHashAggregateExec, SortAggregateExec}
import org.apache.spark.sql.execution.datasources.WriteFilesExec
@@ -34,7 +34,7 @@ import
org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleEx
import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.execution.window.WindowExec
import org.apache.spark.sql.hive.HiveTableScanExecTransformer
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType,
TimestampType}
object Validators {
implicit class ValidatorBuilderImplicits(builder: Validator.Builder) {
@@ -257,6 +257,7 @@ object Validators {
case mt: MapType => containsNTZ(mt.keyType) ||
containsNTZ(mt.valueType)
case _ => false
}
+ def isNTZ(dataType: DataType): Boolean = dataType.typeName ==
"timestamp_ntz"
val hasNTZ = plan.output.exists(a => containsNTZ(a.dataType)) ||
plan.children.exists(_.output.exists(a => containsNTZ(a.dataType)))
if (!hasNTZ) {
@@ -282,6 +283,8 @@ object Validators {
case Minute(child, _) => containsNTZ(child.dataType)
case Second(child, _) => containsNTZ(child.dataType)
case TimestampAdd(_, _, child, _) =>
containsNTZ(child.dataType)
+ case c: Cast if c.dataType == TimestampType =>
isNTZ(c.child.dataType)
+ case c: Cast if isNTZ(c.dataType) => c.child.dataType ==
TimestampType
case _ => false
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
