This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new f4e53fa9910 [SPARK-43336][SQL] Casting between Timestamp and
TimestampNTZ requires timezone
f4e53fa9910 is described below
commit f4e53fa9910780934b49b2fcf6a22dba7b337a11
Author: Gengliang Wang <gengli...@apache.org>
AuthorDate: Tue May 2 13:09:45 2023 -0700
[SPARK-43336][SQL] Casting between Timestamp and TimestampNTZ requires
timezone
### What changes were proposed in this pull request?
Casting between Timestamp and TimestampNTZ requires a timezone since the
timezone id is used in the evaluation.
This PR updates the method `Cast.needsTimeZone` to include the conversion
between Timestamp and TimestampNTZ. As a result:
* Casting between Timestamp and TimestampNTZ is considered as unresolved
unless the timezone is defined
* Canonicalizing cast will include the time zone
### Why are the changes needed?
Timezone id is used in the evaluation between Timestamp and TimestampNTZ,
thus we should mark such conversion as "needsTimeZone"
### Does this PR introduce _any_ user-facing change?
No. It is more like a fix for potential bugs that the casting between
Timestamp and TimestampNTZ is marked as resolved before resolving the timezone.
### How was this patch tested?
New UT
Closes #41010 from gengliangwang/fixCast.
Authored-by: Gengliang Wang <gengli...@apache.org>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
(cherry picked from commit a1c7035294ecdc10f507c58a41cb0ed7137717ca)
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
.../org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 ++
.../spark/sql/catalyst/expressions/CanonicalizeSuite.scala | 14 +++++++++++++-
.../spark/sql/catalyst/expressions/CastSuiteBase.scala | 7 +++++++
3 files changed, 22 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index ae0dc3dbf03..3a93cc58a5f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -271,6 +271,8 @@ object Cast extends QueryErrorsBase {
case (TimestampType | DateType, StringType) => true
case (DateType, TimestampType) => true
case (TimestampType, DateType) => true
+ case (TimestampType, TimestampNTZType) => true
+ case (TimestampNTZType, TimestampType) => true
case (ArrayType(fromType, _), ArrayType(toType, _)) =>
needsTimeZone(fromType, toType)
case (MapType(fromKey, fromValue, _), MapType(toKey, toValue, _)) =>
needsTimeZone(fromKey, toKey) || needsTimeZone(fromValue, toValue)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index f2a9eac8216..0e22b0d2876 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.Range
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.MULTI_COMMUTATIVE_OP_OPT_THRESHOLD
-import org.apache.spark.sql.types.{BooleanType, Decimal, DecimalType,
IntegerType, LongType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{BooleanType, Decimal, DecimalType,
IntegerType, LongType, StringType, StructField, StructType, TimestampNTZType,
TimestampType}
class CanonicalizeSuite extends SparkFunSuite {
@@ -99,6 +99,18 @@ class CanonicalizeSuite extends SparkFunSuite {
assert(castWithTimeZoneId.semanticEquals(cast))
}
+ test("SPARK-43336: Canonicalize Cast between Timestamp and TimestampNTZ
should consider " +
+ "timezone") {
+ val timestampLiteral = Literal.create(1L, TimestampType)
+ val timestampNTZLiteral = Literal.create(1L, TimestampNTZType)
+ Seq(
+ Cast(timestampLiteral, TimestampNTZType),
+ Cast(timestampNTZLiteral, TimestampType)
+ ).foreach { cast =>
+
assert(!cast.semanticEquals(cast.withTimeZone(SQLConf.get.sessionLocalTimeZone)))
+ }
+ }
+
test("SPARK-32927: Bitwise operations are commutative") {
Seq(BitwiseOr(_, _), BitwiseAnd(_, _), BitwiseXor(_, _)).foreach { f =>
val e1 = f($"a", f($"b", $"c"))
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index bad85ca4176..7c11ccf6a0d 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -1391,4 +1391,11 @@ abstract class CastSuiteBase extends SparkFunSuite with
ExpressionEvalHelper {
assert(expr.sql == cast.sql)
assert(expr.toString == cast.toString)
}
+
+ test("SPARK-43336: Casting between Timestamp and TimestampNTZ requires
timezone") {
+ val timestampLiteral = Literal.create(1L, TimestampType)
+ val timestampNTZLiteral = Literal.create(1L, TimestampNTZType)
+ assert(!Cast(timestampLiteral, TimestampNTZType).resolved)
+ assert(!Cast(timestampNTZLiteral, TimestampType).resolved)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
