This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 618b52097c0 [SPARK-44030][SQL][FOLLOW-UP] Move unapply from AnyTimestampType to AnyTimestampTypeExpression 618b52097c0 is described below commit 618b52097c07105d734aaf9b2a22b372920b3f31 Author: Rui Wang <rui.w...@databricks.com> AuthorDate: Fri Jun 30 08:38:39 2023 +0300 [SPARK-44030][SQL][FOLLOW-UP] Move unapply from AnyTimestampType to AnyTimestampTypeExpression ### What changes were proposed in this pull request? Move unapply from AnyTimestampType to AnyTimestampTypeExpression. ### Why are the changes needed? To align with the effort that we use separate type expression class to host `unapply`. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing Test Closes #41771 from amaliujia/atomic_datatype_expression. Authored-by: Rui Wang <rui.w...@databricks.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 4 ++-- .../spark/sql/catalyst/analysis/AnsiTypeCoercion.scala | 14 ++++++++------ .../apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 12 +++++++----- .../org/apache/spark/sql/types/AbstractDataType.scala | 3 --- .../org/apache/spark/sql/types/DataTypeExpression.scala | 5 +++++ 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 8a192a4c132..32cec909401 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -428,8 +428,8 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor UnaryMinus(r, mode == EvalMode.ANSI), ansiEnabled = mode == EvalMode.ANSI)) case (_, CalendarIntervalType | _: DayTimeIntervalType) => Cast(DatetimeSub(l, r, TimeAdd(l, UnaryMinus(r, mode == EvalMode.ANSI))), l.dataType) - case _ if AnyTimestampType.unapply(l) || AnyTimestampType.unapply(r) => - SubtractTimestamps(l, r) + case _ if AnyTimestampTypeExpression.unapply(l) || + AnyTimestampTypeExpression.unapply(r) => SubtractTimestamps(l, r) case (_, DateType) => SubtractDates(l, r) case (DateType, dt) if dt != StringType => DateSub(l, r) case _ => s diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala index d3f20f87493..5854f42a061 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala @@ -284,7 +284,7 @@ object AnsiTypeCoercion extends TypeCoercionBase { // Skip nodes who's children have not been resolved yet. case g if !g.childrenResolved => g - case g: GetDateField if AnyTimestampType.unapply(g.child) => + case g: GetDateField if AnyTimestampTypeExpression.unapply(g.child) => g.withNewChildren(Seq(Cast(g.child, DateType))) } } @@ -294,14 +294,16 @@ object AnsiTypeCoercion extends TypeCoercionBase { // Skip nodes who's children have not been resolved yet. case e if !e.childrenResolved => e - case d @ DateAdd(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType)) - case d @ DateSub(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType)) + case d @ DateAdd(AnyTimestampTypeExpression(), _) => + d.copy(startDate = Cast(d.startDate, DateType)) + case d @ DateSub(AnyTimestampTypeExpression(), _) => + d.copy(startDate = Cast(d.startDate, DateType)) - case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampType(), _, _) => + case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampTypeExpression(), _, _) => s.copy(left = Cast(s.left, s.right.dataType)) - case s @ SubtractTimestamps(AnyTimestampType(), DateTypeExpression(), _, _) => + case s @ SubtractTimestamps(AnyTimestampTypeExpression(), DateTypeExpression(), _, _) => s.copy(right = Cast(s.right, s.left.dataType)) - case s @ SubtractTimestamps(AnyTimestampType(), AnyTimestampType(), _, _) + case s @ SubtractTimestamps(AnyTimestampTypeExpression(), AnyTimestampTypeExpression(), _, _) if s.left.dataType != s.right.dataType => val newLeft = castIfNotSameType(s.left, TimestampNTZType) val newRight = castIfNotSameType(s.right, TimestampNTZType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index ae4db0575ad..2a1067be004 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -1174,16 +1174,18 @@ object TypeCoercion extends TypeCoercionBase { override val transform: PartialFunction[Expression, Expression] = { // Skip nodes who's children have not been resolved yet. case e if !e.childrenResolved => e - case d @ DateAdd(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType)) + case d @ DateAdd(AnyTimestampTypeExpression(), _) => + d.copy(startDate = Cast(d.startDate, DateType)) case d @ DateAdd(StringTypeExpression(), _) => d.copy(startDate = Cast(d.startDate, DateType)) - case d @ DateSub(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType)) + case d @ DateSub(AnyTimestampTypeExpression(), _) => + d.copy(startDate = Cast(d.startDate, DateType)) case d @ DateSub(StringTypeExpression(), _) => d.copy(startDate = Cast(d.startDate, DateType)) - case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampType(), _, _) => + case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampTypeExpression(), _, _) => s.copy(left = Cast(s.left, s.right.dataType)) - case s @ SubtractTimestamps(AnyTimestampType(), DateTypeExpression(), _, _) => + case s @ SubtractTimestamps(AnyTimestampTypeExpression(), DateTypeExpression(), _, _) => s.copy(right = Cast(s.right, s.left.dataType)) - case s @ SubtractTimestamps(AnyTimestampType(), AnyTimestampType(), _, _) + case s @ SubtractTimestamps(AnyTimestampTypeExpression(), AnyTimestampTypeExpression(), _, _) if s.left.dataType != s.right.dataType => val newLeft = castIfNotSameType(s.left, TimestampNTZType) val newRight = castIfNotSameType(s.right, TimestampNTZType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala index 01fa27822b0..c1483e719b5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.types import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.errors.QueryExecutionErrors /** @@ -167,8 +166,6 @@ private[sql] object AnyTimestampType extends AbstractDataType with Serializable other.isInstanceOf[TimestampType] || other.isInstanceOf[TimestampNTZType] override private[sql] def simpleString = "(timestamp or timestamp without time zone)" - - def unapply(e: Expression): Boolean = acceptsType(e.dataType) } private[sql] abstract class DatetimeType extends AtomicType diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala index b2455a41ae6..1b74419a4af 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala @@ -66,6 +66,11 @@ object IntegralTypeExpression { } } +object AnyTimestampTypeExpression { + def unapply(e: Expression): Boolean = + e.dataType.isInstanceOf[TimestampType] || e.dataType.isInstanceOf[TimestampNTZType] +} + object DecimalExpression { def unapply(e: Expression): Option[(Int, Int)] = e.dataType match { case t: DecimalType => Some((t.precision, t.scale)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org