This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 44c1c03 [SPARK-29607][SQL] Move static methods from CalendarInterval to IntervalUtils 44c1c03 is described below commit 44c1c03924c9e829f84b0c1344feb699e062a5d7 Author: Maxim Gekk <max.g...@gmail.com> AuthorDate: Wed Oct 30 01:15:18 2019 +0800 [SPARK-29607][SQL] Move static methods from CalendarInterval to IntervalUtils ### What changes were proposed in this pull request? In the PR, I propose to move all static methods from the `CalendarInterval` class to the `IntervalUtils` object. All those methods are rewritten from Java to Scala. ### Why are the changes needed? - For consistency with other helper methods. Such methods were placed to the helper object `IntervalUtils`, see https://github.com/apache/spark/pull/26190 - Taking into account that `CalendarInterval` will be fully exposed to users in the future (see https://github.com/apache/spark/pull/25022), it would be nice to clean it up by moving service methods to an internal object. ### Does this PR introduce any user-facing change? No ### How was this patch tested? - By moved tests from `CalendarIntervalSuite` to `IntervalUtilsSuite` - By existing test suites Closes #26261 from MaxGekk/refactoring-calendar-interval. Authored-by: Maxim Gekk <max.g...@gmail.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../spark/unsafe/types/CalendarInterval.java | 205 --------------------- .../spark/unsafe/types/CalendarIntervalSuite.java | 66 ------- .../spark/sql/catalyst/parser/AstBuilder.scala | 18 +- .../spark/sql/catalyst/util/IntervalUtils.scala | 198 ++++++++++++++++++++ .../expressions/CollectionExpressionsSuite.scala | 6 +- .../catalyst/parser/ExpressionParserSuite.scala | 8 +- .../sql/catalyst/util/IntervalUtilsSuite.scala | 63 ++++++- 7 files changed, 276 insertions(+), 288 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java index 184ddac..3c83551 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java @@ -18,8 +18,6 @@ package org.apache.spark.unsafe.types; import java.io.Serializable; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * The internal representation of interval type. @@ -32,209 +30,6 @@ public final class CalendarInterval implements Serializable { public static final long MICROS_PER_DAY = MICROS_PER_HOUR * 24; public static final long MICROS_PER_WEEK = MICROS_PER_DAY * 7; - private static Pattern yearMonthPattern = Pattern.compile( - "^([+|-])?(\\d+)-(\\d+)$"); - - private static Pattern dayTimePattern = Pattern.compile( - "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$"); - - public static long toLongWithRange(String fieldName, - String s, long minValue, long maxValue) throws IllegalArgumentException { - long result = 0; - if (s != null) { - result = Long.parseLong(s); - if (result < minValue || result > maxValue) { - throw new IllegalArgumentException(String.format("%s %d outside range [%d, %d]", - fieldName, result, minValue, maxValue)); - } - } - return result; - } - - /** - * Parse YearMonth string in form: [-]YYYY-MM - * - * adapted from HiveIntervalYearMonth.valueOf - */ - public static CalendarInterval fromYearMonthString(String s) throws IllegalArgumentException { - CalendarInterval result = null; - if (s == null) { - throw new IllegalArgumentException("Interval year-month string was null"); - } - s = s.trim(); - Matcher m = yearMonthPattern.matcher(s); - if (!m.matches()) { - throw new IllegalArgumentException( - "Interval string does not match year-month format of 'y-m': " + s); - } else { - try { - int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1; - int years = (int) toLongWithRange("year", m.group(2), 0, Integer.MAX_VALUE); - int months = (int) toLongWithRange("month", m.group(3), 0, 11); - result = new CalendarInterval(sign * (years * 12 + months), 0); - } catch (Exception e) { - throw new IllegalArgumentException( - "Error parsing interval year-month string: " + e.getMessage(), e); - } - } - return result; - } - - /** - * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn - * - * adapted from HiveIntervalDayTime.valueOf - */ - public static CalendarInterval fromDayTimeString(String s) throws IllegalArgumentException { - return fromDayTimeString(s, "day", "second"); - } - - /** - * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn - * - * adapted from HiveIntervalDayTime.valueOf. - * Below interval conversion patterns are supported: - * - DAY TO (HOUR|MINUTE|SECOND) - * - HOUR TO (MINUTE|SECOND) - * - MINUTE TO SECOND - */ - public static CalendarInterval fromDayTimeString(String s, String from, String to) - throws IllegalArgumentException { - CalendarInterval result = null; - if (s == null) { - throw new IllegalArgumentException("Interval day-time string was null"); - } - s = s.trim(); - Matcher m = dayTimePattern.matcher(s); - if (!m.matches()) { - throw new IllegalArgumentException( - "Interval string does not match day-time format of 'd h:m:s.n': " + s); - } else { - try { - int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1; - long days = m.group(2) == null ? 0 : toLongWithRange("day", m.group(3), - 0, Integer.MAX_VALUE); - long hours = 0; - long minutes; - long seconds = 0; - if (m.group(5) != null || from.equals("minute")) { // 'HH:mm:ss' or 'mm:ss minute' - hours = toLongWithRange("hour", m.group(5), 0, 23); - minutes = toLongWithRange("minute", m.group(6), 0, 59); - seconds = toLongWithRange("second", m.group(7), 0, 59); - } else if (m.group(8) != null){ // 'mm:ss.nn' - minutes = toLongWithRange("minute", m.group(6), 0, 59); - seconds = toLongWithRange("second", m.group(7), 0, 59); - } else { // 'HH:mm' - hours = toLongWithRange("hour", m.group(6), 0, 23); - minutes = toLongWithRange("second", m.group(7), 0, 59); - } - // Hive allow nanosecond precision interval - String nanoStr = m.group(9) == null ? null : (m.group(9) + "000000000").substring(0, 9); - long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L); - switch (to) { - case "hour": - minutes = 0; - seconds = 0; - nanos = 0; - break; - case "minute": - seconds = 0; - nanos = 0; - break; - case "second": - // No-op - break; - default: - throw new IllegalArgumentException( - String.format("Cannot support (interval '%s' %s to %s) expression", s, from, to)); - } - result = new CalendarInterval(0, sign * ( - days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE + - seconds * MICROS_PER_SECOND + nanos / 1000L)); - } catch (Exception e) { - throw new IllegalArgumentException( - "Error parsing interval day-time string: " + e.getMessage(), e); - } - } - return result; - } - - public static CalendarInterval fromUnitStrings(String[] units, String[] values) - throws IllegalArgumentException { - assert units.length == values.length; - int months = 0; - long microseconds = 0; - - for (int i = 0; i < units.length; i++) { - try { - switch (units[i]) { - case "year": - months = Math.addExact(months, Math.multiplyExact(Integer.parseInt(values[i]), 12)); - break; - case "month": - months = Math.addExact(months, Integer.parseInt(values[i])); - break; - case "week": - microseconds = Math.addExact( - microseconds, - Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_WEEK)); - break; - case "day": - microseconds = Math.addExact( - microseconds, - Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_DAY)); - break; - case "hour": - microseconds = Math.addExact( - microseconds, - Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_HOUR)); - break; - case "minute": - microseconds = Math.addExact( - microseconds, - Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_MINUTE)); - break; - case "second": { - microseconds = Math.addExact(microseconds, parseSecondNano(values[i])); - break; - } - case "millisecond": - microseconds = Math.addExact( - microseconds, - Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_MILLI)); - break; - case "microsecond": - microseconds = Math.addExact(microseconds, Long.parseLong(values[i])); - break; - } - } catch (Exception e) { - throw new IllegalArgumentException("Error parsing interval string: " + e.getMessage(), e); - } - } - return new CalendarInterval(months, microseconds); - } - - /** - * Parse second_nano string in ss.nnnnnnnnn format to microseconds - */ - public static long parseSecondNano(String secondNano) throws IllegalArgumentException { - String[] parts = secondNano.split("\\."); - if (parts.length == 1) { - return toLongWithRange("second", parts[0], Long.MIN_VALUE / MICROS_PER_SECOND, - Long.MAX_VALUE / MICROS_PER_SECOND) * MICROS_PER_SECOND; - - } else if (parts.length == 2) { - long seconds = parts[0].equals("") ? 0L : toLongWithRange("second", parts[0], - Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE / MICROS_PER_SECOND); - long nanos = toLongWithRange("nanosecond", parts[1], 0L, 999999999L); - return seconds * MICROS_PER_SECOND + nanos / 1000L; - - } else { - throw new IllegalArgumentException( - "Interval string does not match second-nano format of ss.nnnnnnnnn"); - } - } - public final int months; public final long microseconds; diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java index 9f3262b..5e418c1 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java @@ -61,72 +61,6 @@ public class CalendarIntervalSuite { } @Test - public void fromYearMonthStringTest() { - String input; - CalendarInterval i; - - input = "99-10"; - i = new CalendarInterval(99 * 12 + 10, 0L); - assertEquals(fromYearMonthString(input), i); - - input = "-8-10"; - i = new CalendarInterval(-8 * 12 - 10, 0L); - assertEquals(fromYearMonthString(input), i); - - try { - input = "99-15"; - fromYearMonthString(input); - fail("Expected to throw an exception for the invalid input"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("month 15 outside range")); - } - } - - @Test - public void fromDayTimeStringTest() { - String input; - CalendarInterval i; - - input = "5 12:40:30.999999999"; - i = new CalendarInterval(0, 5 * MICROS_PER_DAY + 12 * MICROS_PER_HOUR + - 40 * MICROS_PER_MINUTE + 30 * MICROS_PER_SECOND + 999999L); - assertEquals(fromDayTimeString(input), i); - - input = "10 0:12:0.888"; - i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE + - 888 * MICROS_PER_MILLI); - assertEquals(fromDayTimeString(input), i); - - input = "-3 0:0:0"; - i = new CalendarInterval(0, -3 * MICROS_PER_DAY); - assertEquals(fromDayTimeString(input), i); - - try { - input = "5 30:12:20"; - fromDayTimeString(input); - fail("Expected to throw an exception for the invalid input"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("hour 30 outside range")); - } - - try { - input = "5 30-12"; - fromDayTimeString(input); - fail("Expected to throw an exception for the invalid input"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("not match day-time format")); - } - - try { - input = "5 1:12:20"; - fromDayTimeString(input, "hour", "microsecond"); - fail("Expected to throw an exception for the invalid convention type"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Cannot support (interval")); - } - } - - @Test public void addTest() { CalendarInterval input1 = new CalendarInterval(3, 1 * MICROS_PER_HOUR); CalendarInterval input2 = new CalendarInterval(2, 100 * MICROS_PER_HOUR); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 893003d..072b9a1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -108,7 +108,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging }.toArray val values = ctx.intervalValue().asScala.map(getIntervalValue).toArray try { - CalendarInterval.fromUnitStrings(units, values) + IntervalUtils.fromUnitStrings(units, values) } catch { case i: IllegalArgumentException => val e = new ParseException(i.getMessage, ctx) @@ -1953,21 +1953,21 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging val unitText = unit.getText.toLowerCase(Locale.ROOT) val interval = (unitText, Option(to).map(_.getText.toLowerCase(Locale.ROOT))) match { case (u, None) => - CalendarInterval.fromUnitStrings(Array(normalizeInternalUnit(u)), Array(s)) + IntervalUtils.fromUnitStrings(Array(normalizeInternalUnit(u)), Array(s)) case ("year", Some("month")) => - CalendarInterval.fromYearMonthString(s) + IntervalUtils.fromYearMonthString(s) case ("day", Some("hour")) => - CalendarInterval.fromDayTimeString(s, "day", "hour") + IntervalUtils.fromDayTimeString(s, "day", "hour") case ("day", Some("minute")) => - CalendarInterval.fromDayTimeString(s, "day", "minute") + IntervalUtils.fromDayTimeString(s, "day", "minute") case ("day", Some("second")) => - CalendarInterval.fromDayTimeString(s, "day", "second") + IntervalUtils.fromDayTimeString(s, "day", "second") case ("hour", Some("minute")) => - CalendarInterval.fromDayTimeString(s, "hour", "minute") + IntervalUtils.fromDayTimeString(s, "hour", "minute") case ("hour", Some("second")) => - CalendarInterval.fromDayTimeString(s, "hour", "second") + IntervalUtils.fromDayTimeString(s, "hour", "second") case ("minute", Some("second")) => - CalendarInterval.fromDayTimeString(s, "minute", "second") + IntervalUtils.fromDayTimeString(s, "minute", "second") case (from, Some(t)) => throw new ParseException(s"Intervals FROM $from TO $t are not supported.", ctx) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 14fd153..f55b054 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -17,6 +17,10 @@ package org.apache.spark.sql.catalyst.util +import java.util.regex.Pattern + +import scala.util.control.NonFatal + import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException} import org.apache.spark.sql.types.Decimal import org.apache.spark.unsafe.types.CalendarInterval @@ -119,4 +123,198 @@ object IntervalUtils { case _: IllegalArgumentException => null } } + + private def toLongWithRange( + fieldName: String, + s: String, + minValue: Long, + maxValue: Long): Long = { + val result = if (s == null) 0L else s.toLong + require(minValue <= result && result <= maxValue, + s"$fieldName $result outside range [$minValue, $maxValue]") + + result + } + + private val yearMonthPattern = "^([+|-])?(\\d+)-(\\d+)$".r + + /** + * Parse YearMonth string in form: [+|-]YYYY-MM + * + * adapted from HiveIntervalYearMonth.valueOf + */ + def fromYearMonthString(input: String): CalendarInterval = { + require(input != null, "Interval year-month string must be not null") + def toInterval(yearStr: String, monthStr: String): CalendarInterval = { + try { + val years = toLongWithRange("year", yearStr, 0, Integer.MAX_VALUE).toInt + val months = toLongWithRange("month", monthStr, 0, 11).toInt + val totalMonths = Math.addExact(Math.multiplyExact(years, 12), months) + new CalendarInterval(totalMonths, 0) + } catch { + case NonFatal(e) => + throw new IllegalArgumentException( + s"Error parsing interval year-month string: ${e.getMessage}", e) + } + } + assert(input.length == input.trim.length) + input match { + case yearMonthPattern("-", yearStr, monthStr) => + toInterval(yearStr, monthStr).negate() + case yearMonthPattern(_, yearStr, monthStr) => + toInterval(yearStr, monthStr) + case _ => + throw new IllegalArgumentException( + s"Interval string does not match year-month format of 'y-m': $input") + } + } + + /** + * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn + * + * adapted from HiveIntervalDayTime.valueOf + */ + def fromDayTimeString(s: String): CalendarInterval = { + fromDayTimeString(s, "day", "second") + } + + private val dayTimePattern = + "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r + + /** + * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn + * + * adapted from HiveIntervalDayTime.valueOf. + * Below interval conversion patterns are supported: + * - DAY TO (HOUR|MINUTE|SECOND) + * - HOUR TO (MINUTE|SECOND) + * - MINUTE TO SECOND + */ + def fromDayTimeString(input: String, from: String, to: String): CalendarInterval = { + require(input != null, "Interval day-time string must be not null") + assert(input.length == input.trim.length) + val m = dayTimePattern.pattern.matcher(input) + require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input") + + try { + val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1 + val days = if (m.group(2) == null) { + 0 + } else { + toLongWithRange("day", m.group(3), 0, Integer.MAX_VALUE) + } + var hours: Long = 0L + var minutes: Long = 0L + var seconds: Long = 0L + if (m.group(5) != null || from == "minute") { // 'HH:mm:ss' or 'mm:ss minute' + hours = toLongWithRange("hour", m.group(5), 0, 23) + minutes = toLongWithRange("minute", m.group(6), 0, 59) + seconds = toLongWithRange("second", m.group(7), 0, 59) + } else if (m.group(8) != null) { // 'mm:ss.nn' + minutes = toLongWithRange("minute", m.group(6), 0, 59) + seconds = toLongWithRange("second", m.group(7), 0, 59) + } else { // 'HH:mm' + hours = toLongWithRange("hour", m.group(6), 0, 23) + minutes = toLongWithRange("second", m.group(7), 0, 59) + } + // Hive allow nanosecond precision interval + val nanoStr = if (m.group(9) == null) { + null + } else { + (m.group(9) + "000000000").substring(0, 9) + } + var nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L) + to match { + case "hour" => + minutes = 0 + seconds = 0 + nanos = 0 + case "minute" => + seconds = 0 + nanos = 0 + case "second" => + // No-op + case _ => + throw new IllegalArgumentException( + s"Cannot support (interval '$input' $from to $to) expression") + } + var micros = nanos / DateTimeUtils.NANOS_PER_MICROS + micros = Math.addExact(micros, Math.multiplyExact(days, DateTimeUtils.MICROS_PER_DAY)) + micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR)) + micros = Math.addExact(micros, Math.multiplyExact(minutes, MICROS_PER_MINUTE)) + micros = Math.addExact(micros, Math.multiplyExact(seconds, DateTimeUtils.MICROS_PER_SECOND)) + new CalendarInterval(0, sign * micros) + } catch { + case e: Exception => + throw new IllegalArgumentException( + s"Error parsing interval day-time string: ${e.getMessage}", e) + } + } + + def fromUnitStrings(units: Array[String], values: Array[String]): CalendarInterval = { + assert(units.length == values.length) + var months: Int = 0 + var microseconds: Long = 0 + var i = 0 + while (i < units.length) { + try { + units(i) match { + case "year" => + months = Math.addExact(months, Math.multiplyExact(values(i).toInt, 12)) + case "month" => + months = Math.addExact(months, values(i).toInt) + case "week" => + val weeksUs = Math.multiplyExact(values(i).toLong, 7 * DateTimeUtils.MICROS_PER_DAY) + microseconds = Math.addExact(microseconds, weeksUs) + case "day" => + val daysUs = Math.multiplyExact(values(i).toLong, DateTimeUtils.MICROS_PER_DAY) + microseconds = Math.addExact(microseconds, daysUs) + case "hour" => + val hoursUs = Math.multiplyExact(values(i).toLong, MICROS_PER_HOUR) + microseconds = Math.addExact(microseconds, hoursUs) + case "minute" => + val minutesUs = Math.multiplyExact(values(i).toLong, MICROS_PER_MINUTE) + microseconds = Math.addExact(microseconds, minutesUs) + case "second" => + microseconds = Math.addExact(microseconds, parseSecondNano(values(i))) + case "millisecond" => + val millisUs = Math.multiplyExact(values(i).toLong, DateTimeUtils.MICROS_PER_MILLIS) + microseconds = Math.addExact(microseconds, millisUs) + case "microsecond" => + microseconds = Math.addExact(microseconds, values(i).toLong) + } + } catch { + case e: Exception => + throw new IllegalArgumentException(s"Error parsing interval string: ${e.getMessage}", e) + } + i += 1 + } + new CalendarInterval(months, microseconds) + } + + /** + * Parse second_nano string in ss.nnnnnnnnn format to microseconds + */ + private def parseSecondNano(secondNano: String): Long = { + def parseSeconds(secondsStr: String): Long = { + toLongWithRange( + "second", + secondsStr, + Long.MinValue / DateTimeUtils.MICROS_PER_SECOND, + Long.MaxValue / DateTimeUtils.MICROS_PER_SECOND) * DateTimeUtils.MICROS_PER_SECOND + } + def parseNanos(nanosStr: String): Long = { + toLongWithRange("nanosecond", nanosStr, 0L, 999999999L) / DateTimeUtils.NANOS_PER_MICROS + } + + secondNano.split("\\.") match { + case Array(secondsStr) => parseSeconds(secondsStr) + case Array("", nanosStr) => parseNanos(nanosStr) + case Array(secondsStr, nanosStr) => + Math.addExact(parseSeconds(secondsStr), parseNanos(nanosStr)) + case _ => + throw new IllegalArgumentException( + "Interval string does not match second-nano format of ss.nnnnnnnnn") + } + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index e10aa60..74d1606 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -810,7 +810,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(new Sequence( Literal(Timestamp.valueOf("2018-01-01 00:00:00")), Literal(Timestamp.valueOf("2023-01-01 00:00:00")), - Literal(CalendarInterval.fromYearMonthString("1-5"))), + Literal(IntervalUtils.fromYearMonthString("1-5"))), Seq( Timestamp.valueOf("2018-01-01 00:00:00.000"), Timestamp.valueOf("2019-06-01 00:00:00.000"), @@ -820,7 +820,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(new Sequence( Literal(Timestamp.valueOf("2022-04-01 00:00:00")), Literal(Timestamp.valueOf("2017-01-01 00:00:00")), - Literal(CalendarInterval.fromYearMonthString("1-5").negate())), + Literal(IntervalUtils.fromYearMonthString("1-5").negate())), Seq( Timestamp.valueOf("2022-04-01 00:00:00.000"), Timestamp.valueOf("2020-11-01 00:00:00.000"), @@ -894,7 +894,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(new Sequence( Literal(Date.valueOf("2018-01-01")), Literal(Date.valueOf("2023-01-01")), - Literal(CalendarInterval.fromYearMonthString("1-5"))), + Literal(IntervalUtils.fromYearMonthString("1-5"))), Seq( Date.valueOf("2018-01-01"), Date.valueOf("2019-06-01"), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 86b3aa81..5a7b3ff 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -597,7 +597,7 @@ class ExpressionParserSuite extends AnalysisTest { "microsecond") def intervalLiteral(u: String, s: String): Literal = { - Literal(CalendarInterval.fromUnitStrings(Array(u), Array(s))) + Literal(IntervalUtils.fromUnitStrings(Array(u), Array(s))) } test("intervals") { @@ -637,7 +637,7 @@ class ExpressionParserSuite extends AnalysisTest { // Year-Month intervals. val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0") yearMonthValues.foreach { value => - val result = Literal(CalendarInterval.fromYearMonthString(value)) + val result = Literal(IntervalUtils.fromYearMonthString(value)) checkIntervals(s"'$value' year to month", result) } @@ -650,7 +650,7 @@ class ExpressionParserSuite extends AnalysisTest { "-1 0:0:0", "1 0:0:1") datTimeValues.foreach { value => - val result = Literal(CalendarInterval.fromDayTimeString(value)) + val result = Literal(IntervalUtils.fromDayTimeString(value)) checkIntervals(s"'$value' day to second", result) } @@ -662,7 +662,7 @@ class ExpressionParserSuite extends AnalysisTest { "0:0:0", "0:0:1") hourTimeValues.foreach { value => - val result = Literal(CalendarInterval.fromDayTimeString(value)) + val result = Literal(IntervalUtils.fromDayTimeString(value)) checkIntervals(s"'$value' hour to second", result) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala index e48779a..9addc39 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.util import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.util.IntervalUtils.fromString +import org.apache.spark.sql.catalyst.util.IntervalUtils.{fromDayTimeString, fromString, fromYearMonthString} import org.apache.spark.unsafe.types.CalendarInterval import org.apache.spark.unsafe.types.CalendarInterval._ @@ -87,4 +87,65 @@ class IntervalUtilsSuite extends SparkFunSuite { assert(fromString(input2) == result) } } + + test("from year-month string") { + assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10, 0L)) + assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 + 10, 0L)) + assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10, 0L)) + + try { + fromYearMonthString("99-15") + fail("Expected to throw an exception for the invalid input") + } catch { + case e: IllegalArgumentException => + assert(e.getMessage.contains("month 15 outside range")) + } + + try { + fromYearMonthString("9a9-15") + fail("Expected to throw an exception for the invalid input") + } catch { + case e: IllegalArgumentException => + assert(e.getMessage.contains("Interval string does not match year-month format")) + } + } + + test("from day-time string") { + assert(fromDayTimeString("5 12:40:30.999999999") === + new CalendarInterval( + 0, + 5 * MICROS_PER_DAY + + 12 * MICROS_PER_HOUR + + 40 * MICROS_PER_MINUTE + + 30 * MICROS_PER_SECOND + 999999L)) + assert(fromDayTimeString("10 0:12:0.888") === + new CalendarInterval( + 0, + 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE + 888 * MICROS_PER_MILLI)) + assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3 * MICROS_PER_DAY)) + + try { + fromDayTimeString("5 30:12:20") + fail("Expected to throw an exception for the invalid input") + } catch { + case e: IllegalArgumentException => + assert(e.getMessage.contains("hour 30 outside range")) + } + + try { + fromDayTimeString("5 30-12") + fail("Expected to throw an exception for the invalid input") + } catch { + case e: IllegalArgumentException => + assert(e.getMessage.contains("must match day-time format")) + } + + try { + fromDayTimeString("5 1:12:20", "hour", "microsecond") + fail("Expected to throw an exception for the invalid convention type") + } catch { + case e: IllegalArgumentException => + assert(e.getMessage.contains("Cannot support (interval")) + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org