This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 2776e8a [SPARK-36021][SQL][FOLLOWUP] DT/YM func use field byte to keep consistence 2776e8a is described below commit 2776e8aa4792a7b2e95d5bbbd76cea0f9554503c Author: Angerszhuuuu <angers....@gmail.com> AuthorDate: Thu Jul 8 12:22:04 2021 +0300 [SPARK-36021][SQL][FOLLOWUP] DT/YM func use field byte to keep consistence ### What changes were proposed in this pull request? With more thought, all DT/YM function use field byte to keep consistence is better ### Why are the changes needed? Keep code consistence ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Not need Closes #33252 from AngersZhuuuu/SPARK-36021-FOLLOWUP. Authored-by: Angerszhuuuu <angers....@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> (cherry picked from commit 89aa16b4a838246cfb7bdc9318461485016f6252) Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../spark/sql/catalyst/parser/AstBuilder.scala | 17 ++++----------- .../spark/sql/catalyst/util/IntervalUtils.scala | 24 ++++++++++------------ .../catalyst/parser/ExpressionParserSuite.scala | 3 ++- .../sql/catalyst/util/IntervalUtilsSuite.scala | 13 ++++-------- 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index d6363b5..4f1e53f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -40,7 +40,6 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.trees.CurrentOrigin import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils} import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate, convertSpecialTimestamp, convertSpecialTimestampNTZ, getZoneId, stringToDate, stringToTimestamp, stringToTimestampWithoutTimeZone} -import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit import org.apache.spark.sql.connector.catalog.{SupportsNamespaces, TableCatalog} import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, Expression => V2Expression, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform} @@ -2487,18 +2486,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg (from, to) match { case ("year", "month") => IntervalUtils.fromYearMonthString(value) - case ("day", "hour") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.HOUR) - case ("day", "minute") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.MINUTE) - case ("day", "second") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.SECOND) - case ("hour", "minute") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.HOUR, IntervalUnit.MINUTE) - case ("hour", "second") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.HOUR, IntervalUnit.SECOND) - case ("minute", "second") => - IntervalUtils.fromDayTimeString(value, IntervalUnit.MINUTE, IntervalUnit.SECOND) + case ("day", "hour") | ("day", "minute") | ("day", "second") | ("hour", "minute") | + ("hour", "second") | ("minute", "second") => + IntervalUtils.fromDayTimeString(value, + DayTimeIntervalType.stringToField(from), DayTimeIntervalType.stringToField(to)) case _ => throw QueryParsingErrors.fromToIntervalUnsupportedError(from, to, ctx) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 24bcad8..7579a28 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -458,7 +458,7 @@ object IntervalUtils { * adapted from HiveIntervalDayTime.valueOf */ def fromDayTimeString(s: String): CalendarInterval = { - fromDayTimeString(s, DAY, SECOND) + fromDayTimeString(s, DT.DAY, DT.SECOND) } /** @@ -470,13 +470,12 @@ object IntervalUtils { * - HOUR TO (HOUR|MINUTE|SECOND) * - MINUTE TO (MINUTE|SECOND) */ - def fromDayTimeString(input: String, from: IntervalUnit, to: IntervalUnit): CalendarInterval = { + def fromDayTimeString(input: String, from: Byte, to: Byte): CalendarInterval = { require(input != null, "Interval day-time string must be not null") if (SQLConf.get.getConf(SQLConf.LEGACY_FROM_DAYTIME_STRING)) { parseDayTimeLegacy(input, from, to) } else { - castDayTimeStringToInterval( - input, DT.stringToField(from.toString), DT.stringToField(to.toString)) + castDayTimeStringToInterval(input, from, to) } } @@ -500,8 +499,8 @@ object IntervalUtils { */ private def parseDayTimeLegacy( input: String, - from: IntervalUnit, - to: IntervalUnit): CalendarInterval = { + from: Byte, + to: Byte): CalendarInterval = { assert(input.length == input.trim.length) val m = dayTimePatternLegacy.pattern.matcher(input) require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " + @@ -517,7 +516,7 @@ object IntervalUtils { var hours: Long = 0L var minutes: Long = 0L var seconds: Long = 0L - if (m.group(5) != null || from == MINUTE) { // 'HH:mm:ss' or 'mm:ss minute' + if (m.group(5) != null || from == DT.MINUTE) { // 'HH:mm:ss' or 'mm:ss minute' hours = toLongWithRange(HOUR, m.group(5), 0, 23) minutes = toLongWithRange(MINUTE, m.group(6), 0, 59) seconds = toLongWithRange(SECOND, m.group(7), 0, 59) @@ -531,18 +530,17 @@ object IntervalUtils { // Hive allow nanosecond precision interval var secondsFraction = parseNanos(m.group(9), seconds < 0) to match { - case HOUR => + case DT.HOUR => minutes = 0 seconds = 0 secondsFraction = 0 - case MINUTE => + case DT.MINUTE => seconds = 0 secondsFraction = 0 - case SECOND => + case DT.SECOND => // No-op - case _ => - throw new IllegalArgumentException( - s"Cannot support (interval '$input' $from to $to) expression") + case _ => throw new IllegalArgumentException(s"Cannot support (" + + s"interval '$input' ${DT.fieldToString(from)} to ${DT.fieldToString(to)}) expression") } var micros = secondsFraction micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 818afc4..aa1a451 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -770,7 +770,8 @@ class ExpressionParserSuite extends AnalysisTest { "0:0:0", "0:0:1") hourTimeValues.foreach { value => - val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND)) + val result = Literal(IntervalUtils.fromDayTimeString( + value, DayTimeIntervalType.HOUR, DayTimeIntervalType.SECOND)) checkIntervals(s"'$value' hour to second", result) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index 93e3ead..62059c0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE, HIVE_STYLE} import org.apache.spark.sql.catalyst.util.IntervalUtils._ -import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DayTimeIntervalType import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} @@ -203,8 +202,6 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper { failFuncWithInvalidInput("5 30:12:20", "hour 30 outside range", fromDayTimeString) failFuncWithInvalidInput("5 30-12", "must match day-time format", fromDayTimeString) - failFuncWithInvalidInput("5 1:12:20", "Cannot support (interval", - s => fromDayTimeString(s, HOUR, MICROSECOND)) } } @@ -314,13 +311,14 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper { } test("from day-time string") { - def check(input: String, from: IntervalUnit, to: IntervalUnit, expected: String): Unit = { + import org.apache.spark.sql.types.DayTimeIntervalType._ + def check(input: String, from: Byte, to: Byte, expected: String): Unit = { withClue(s"from = $from, to = $to") { val expectedUtf8 = UTF8String.fromString(expected) assert(fromDayTimeString(input, from, to) === safeStringToInterval(expectedUtf8)) } } - def checkFail(input: String, from: IntervalUnit, to: IntervalUnit, errMsg: String): Unit = { + def checkFail(input: String, from: Byte, to: Byte, errMsg: String): Unit = { failFuncWithInvalidInput(input, errMsg, s => fromDayTimeString(s, from, to)) } @@ -358,10 +356,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper { checkFail("5 30:12:20", DAY, SECOND, "hour 30 outside range") checkFail("5 30-12", DAY, SECOND, "Interval string does not match day-time format") - withClue("Expected to throw an exception for the invalid input") { - val e = intercept[NoSuchElementException](fromDayTimeString("5 1:12:20", HOUR, MICROSECOND)) - assert(e.getMessage.contains("key not found: microsecond")) - } + // whitespaces check("\t +5 12:40\t ", DAY, MINUTE, "5 days 12 hours 40 minutes") checkFail("+5\t 12:40", DAY, MINUTE, "Interval string does not match day-time format") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org