This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 2f54d9e [SPARK-36049][SQL] Remove IntervalUnit 2f54d9e is described below commit 2f54d9eed6dce5e9fc853ef4dceb1abb2b338a34 Author: Angerszhuuuu <angers....@gmail.com> AuthorDate: Thu Jul 8 23:02:21 2021 +0300 [SPARK-36049][SQL] Remove IntervalUnit ### What changes were proposed in this pull request? Remove IntervalUnit ### Why are the changes needed? Clean code ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Not need Closes #33265 from AngersZhuuuu/SPARK-36049. Lead-authored-by: Angerszhuuuu <angers....@gmail.com> Co-authored-by: Maxim Gekk <max.g...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> (cherry picked from commit fef7e1703c342165000f89b01112a8a28a936436) Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../spark/sql/catalyst/util/IntervalUtils.scala | 120 ++++++++------------- .../catalyst/parser/ExpressionParserSuite.scala | 33 ++---- 2 files changed, 58 insertions(+), 95 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 7579a28..e026266 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -41,22 +41,6 @@ object IntervalStringStyles extends Enumeration { object IntervalUtils { - object IntervalUnit extends Enumeration { - type IntervalUnit = Value - - val NANOSECOND = Value(0, "nanosecond") - val MICROSECOND = Value(1, "microsecond") - val MILLISECOND = Value(2, "millisecond") - val SECOND = Value(3, "second") - val MINUTE = Value(4, "minute") - val HOUR = Value(5, "hour") - val DAY = Value(6, "day") - val WEEK = Value(7, "week") - val MONTH = Value(8, "month") - val YEAR = Value(9, "year") - } - import IntervalUnit._ - private val MAX_DAY = Long.MaxValue / MICROS_PER_DAY private val MAX_HOUR = Long.MaxValue / MICROS_PER_HOUR private val MAX_MINUTE = Long.MaxValue / MICROS_PER_MINUTE @@ -97,7 +81,7 @@ object IntervalUtils { def getSeconds(interval: CalendarInterval): Decimal = getSeconds(interval.microseconds) private def toLongWithRange( - fieldName: IntervalUnit, + fieldName: UTF8String, s: String, minValue: Long, maxValue: Long): Long = { @@ -250,10 +234,11 @@ object IntervalUtils { } } - private def toYMInterval(yearStr: String, monthStr: String, sign: Int): Int = { + private def toYMInterval(year: String, month: String, sign: Int): Int = { safeToInterval("year-month") { - val years = toLongWithRange(YEAR, yearStr, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR) - val totalMonths = sign * (years * MONTHS_PER_YEAR + toLongWithRange(MONTH, monthStr, 0, 11)) + val years = toLongWithRange(yearStr, year, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR) + val totalMonths = + sign * (years * MONTHS_PER_YEAR + toLongWithRange(monthStr, month, 0, 11)) Math.toIntExact(totalMonths) } } @@ -402,45 +387,33 @@ object IntervalUtils { } } - def toDTInterval( - dayStr: String, - hourStr: String, - minuteStr: String, - secondStr: String, - sign: Int): Long = { + def toDTInterval(day: String, hour: String, minute: String, second: String, sign: Int): Long = { var micros = 0L - val days = toLongWithRange(DAY, dayStr, 0, MAX_DAY).toInt + val days = toLongWithRange(dayStr, day, 0, MAX_DAY).toInt micros = Math.addExact(micros, sign * days * MICROS_PER_DAY) - val hours = toLongWithRange(HOUR, hourStr, 0, 23) + val hours = toLongWithRange(hourStr, hour, 0, 23) micros = Math.addExact(micros, sign * hours * MICROS_PER_HOUR) - val minutes = toLongWithRange(MINUTE, minuteStr, 0, 59) + val minutes = toLongWithRange(minuteStr, minute, 0, 59) micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE) - micros = Math.addExact(micros, sign * parseSecondNano(secondStr)) + micros = Math.addExact(micros, sign * parseSecondNano(second)) micros } - def toDTInterval( - hourStr: String, - minuteStr: String, - secondStr: String, - sign: Int): Long = { + def toDTInterval(hour: String, minute: String, second: String, sign: Int): Long = { var micros = 0L - val hours = toLongWithRange(HOUR, hourStr, 0, MAX_HOUR) + val hours = toLongWithRange(hourStr, hour, 0, MAX_HOUR) micros = Math.addExact(micros, sign * hours * MICROS_PER_HOUR) - val minutes = toLongWithRange(MINUTE, minuteStr, 0, 59) + val minutes = toLongWithRange(minuteStr, minute, 0, 59) micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE) - micros = Math.addExact(micros, sign * parseSecondNano(secondStr)) + micros = Math.addExact(micros, sign * parseSecondNano(second)) micros } - def toDTInterval( - minuteStr: String, - secondStr: String, - sign: Int): Long = { + def toDTInterval(minute: String, second: String, sign: Int): Long = { var micros = 0L - val minutes = toLongWithRange(MINUTE, minuteStr, 0, MAX_MINUTE) + val minutes = toLongWithRange(minuteStr, minute, 0, MAX_MINUTE) micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE) - micros = Math.addExact(micros, sign * parseSecondNano(secondStr)) + micros = Math.addExact(micros, sign * parseSecondNano(second)) micros } @@ -511,21 +484,21 @@ object IntervalUtils { val days = if (m.group(2) == null) { 0 } else { - toLongWithRange(DAY, m.group(3), 0, Integer.MAX_VALUE).toInt + toLongWithRange(dayStr, m.group(3), 0, Integer.MAX_VALUE).toInt } var hours: Long = 0L var minutes: Long = 0L var seconds: Long = 0L if (m.group(5) != null || from == DT.MINUTE) { // 'HH:mm:ss' or 'mm:ss minute' - hours = toLongWithRange(HOUR, m.group(5), 0, 23) - minutes = toLongWithRange(MINUTE, m.group(6), 0, 59) - seconds = toLongWithRange(SECOND, m.group(7), 0, 59) + hours = toLongWithRange(hourStr, m.group(5), 0, 23) + minutes = toLongWithRange(minuteStr, m.group(6), 0, 59) + seconds = toLongWithRange(secondStr, m.group(7), 0, 59) } else if (m.group(8) != null) { // 'mm:ss.nn' - minutes = toLongWithRange(MINUTE, m.group(6), 0, 59) - seconds = toLongWithRange(SECOND, m.group(7), 0, 59) + minutes = toLongWithRange(minuteStr, m.group(6), 0, 59) + seconds = toLongWithRange(secondStr, m.group(7), 0, 59) } else { // 'HH:mm' - hours = toLongWithRange(HOUR, m.group(6), 0, 23) - minutes = toLongWithRange(SECOND, m.group(7), 0, 59) + hours = toLongWithRange(hourStr, m.group(6), 0, 23) + minutes = toLongWithRange(secondStr, m.group(7), 0, 59) } // Hive allow nanosecond precision interval var secondsFraction = parseNanos(m.group(9), seconds < 0) @@ -555,15 +528,15 @@ object IntervalUtils { } // Parses a string with nanoseconds, truncates the result and returns microseconds - private def parseNanos(nanosStr: String, isNegative: Boolean): Long = { - if (nanosStr != null) { + private def parseNanos(nanos: String, isNegative: Boolean): Long = { + if (nanos != null) { val maxNanosLen = 9 - val alignedStr = if (nanosStr.length < maxNanosLen) { - (nanosStr + "000000000").substring(0, maxNanosLen) - } else nanosStr - val nanos = toLongWithRange(NANOSECOND, alignedStr, 0L, 999999999L) - val micros = nanos / NANOS_PER_MICROS - if (isNegative) -micros else micros + val alignedStr = if (nanos.length < maxNanosLen) { + (nanos + "000000000").substring(0, maxNanosLen) + } else nanos + val nanoSecond = toLongWithRange(nanosStr, alignedStr, 0L, 999999999L) + val microSecond = nanoSecond / NANOS_PER_MICROS + if (isNegative) -microSecond else microSecond } else { 0L } @@ -574,7 +547,7 @@ object IntervalUtils { */ private def parseSecondNano(secondNano: String): Long = { def parseSeconds(secondsStr: String): Long = { - toLongWithRange(SECOND, secondsStr, MIN_SECOND, MAX_SECOND) * MICROS_PER_SECOND + toLongWithRange(secondStr, secondsStr, MIN_SECOND, MAX_SECOND) * MICROS_PER_SECOND } secondNano.split("\\.") match { @@ -774,19 +747,20 @@ object IntervalUtils { UNIT_SUFFIX, UNIT_END = Value } - private final val intervalStr = UTF8String.fromString("interval") - private def unitToUtf8(unit: IntervalUnit): UTF8String = { - UTF8String.fromString(unit.toString) + private final val intervalStr = unitToUtf8("interval") + private def unitToUtf8(unit: String): UTF8String = { + UTF8String.fromString(unit) } - private final val yearStr = unitToUtf8(YEAR) - private final val monthStr = unitToUtf8(MONTH) - private final val weekStr = unitToUtf8(WEEK) - private final val dayStr = unitToUtf8(DAY) - private final val hourStr = unitToUtf8(HOUR) - private final val minuteStr = unitToUtf8(MINUTE) - private final val secondStr = unitToUtf8(SECOND) - private final val millisStr = unitToUtf8(MILLISECOND) - private final val microsStr = unitToUtf8(MICROSECOND) + private final val yearStr = unitToUtf8("year") + private final val monthStr = unitToUtf8("month") + private final val weekStr = unitToUtf8("week") + private final val dayStr = unitToUtf8("day") + private final val hourStr = unitToUtf8("hour") + private final val minuteStr = unitToUtf8("minute") + private final val secondStr = unitToUtf8("second") + private final val millisStr = unitToUtf8("millisecond") + private final val microsStr = unitToUtf8("microsecond") + private final val nanosStr = unitToUtf8("nanosecond") /** * A safe version of `stringToInterval`. It returns null for invalid input string. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index aa1a451..de0ff96 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last} import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils} -import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.TimestampTypes import org.apache.spark.sql.types._ @@ -679,18 +678,7 @@ class ExpressionParserSuite extends AnalysisTest { } } - val intervalUnits = Seq( - YEAR, - MONTH, - WEEK, - DAY, - HOUR, - MINUTE, - SECOND, - MILLISECOND, - MICROSECOND) - - def intervalLiteral(u: IntervalUnit, s: String): Literal = { + def intervalLiteral(u: UTF8String, s: String): Literal = { Literal(IntervalUtils.stringToInterval(s + " " + u.toString)) } @@ -710,18 +698,19 @@ class ExpressionParserSuite extends AnalysisTest { // Single Intervals. val forms = Seq("", "s") val values = Seq("0", "10", "-7", "21") - intervalUnits.foreach { unit => - forms.foreach { form => - values.foreach { value => - val expected = intervalLiteral(unit, value) - checkIntervals(s"$value $unit$form", expected) - checkIntervals(s"'$value' $unit$form", expected) - } + Seq("year", "month", "week", "day", "hour", "minute", "second", "millisecond", "microsecond") + .foreach { unit => + forms.foreach { form => + values.foreach { value => + val expected = intervalLiteral(unit, value) + checkIntervals(s"$value $unit$form", expected) + checkIntervals(s"'$value' $unit$form", expected) + } + } } - } // Hive nanosecond notation. - checkIntervals("13.123456789 seconds", intervalLiteral(SECOND, "13.123456789")) + checkIntervals("13.123456789 seconds", intervalLiteral("second", "13.123456789")) checkIntervals( "-13.123456789 second", Literal(new CalendarInterval( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org