This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 44c1c03 [SPARK-29607][SQL] Move static methods from CalendarInterval
to IntervalUtils
44c1c03 is described below
commit 44c1c03924c9e829f84b0c1344feb699e062a5d7
Author: Maxim Gekk <max.g...@gmail.com>
AuthorDate: Wed Oct 30 01:15:18 2019 +0800
[SPARK-29607][SQL] Move static methods from CalendarInterval to
IntervalUtils
### What changes were proposed in this pull request?
In the PR, I propose to move all static methods from the `CalendarInterval`
class to the `IntervalUtils` object. All those methods are rewritten from Java
to Scala.
### Why are the changes needed?
- For consistency with other helper methods. Such methods were placed to
the helper object `IntervalUtils`, see
https://github.com/apache/spark/pull/26190
- Taking into account that `CalendarInterval` will be fully exposed to
users in the future (see https://github.com/apache/spark/pull/25022), it would
be nice to clean it up by moving service methods to an internal object.
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
- By moved tests from `CalendarIntervalSuite` to `IntervalUtilsSuite`
- By existing test suites
Closes #26261 from MaxGekk/refactoring-calendar-interval.
Authored-by: Maxim Gekk <max.g...@gmail.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../spark/unsafe/types/CalendarInterval.java | 205 ---------------------
.../spark/unsafe/types/CalendarIntervalSuite.java | 66 -------
.../spark/sql/catalyst/parser/AstBuilder.scala | 18 +-
.../spark/sql/catalyst/util/IntervalUtils.scala | 198 ++++++++++++++++++++
.../expressions/CollectionExpressionsSuite.scala | 6 +-
.../catalyst/parser/ExpressionParserSuite.scala | 8 +-
.../sql/catalyst/util/IntervalUtilsSuite.scala | 63 ++++++-
7 files changed, 276 insertions(+), 288 deletions(-)
diff --git
a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
index 184ddac..3c83551 100644
---
a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
+++
b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
@@ -18,8 +18,6 @@
package org.apache.spark.unsafe.types;
import java.io.Serializable;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
/**
* The internal representation of interval type.
@@ -32,209 +30,6 @@ public final class CalendarInterval implements Serializable
{
public static final long MICROS_PER_DAY = MICROS_PER_HOUR * 24;
public static final long MICROS_PER_WEEK = MICROS_PER_DAY * 7;
- private static Pattern yearMonthPattern = Pattern.compile(
- "^([+|-])?(\\d+)-(\\d+)$");
-
- private static Pattern dayTimePattern = Pattern.compile(
- "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$");
-
- public static long toLongWithRange(String fieldName,
- String s, long minValue, long maxValue) throws IllegalArgumentException {
- long result = 0;
- if (s != null) {
- result = Long.parseLong(s);
- if (result < minValue || result > maxValue) {
- throw new IllegalArgumentException(String.format("%s %d outside range
[%d, %d]",
- fieldName, result, minValue, maxValue));
- }
- }
- return result;
- }
-
- /**
- * Parse YearMonth string in form: [-]YYYY-MM
- *
- * adapted from HiveIntervalYearMonth.valueOf
- */
- public static CalendarInterval fromYearMonthString(String s) throws
IllegalArgumentException {
- CalendarInterval result = null;
- if (s == null) {
- throw new IllegalArgumentException("Interval year-month string was
null");
- }
- s = s.trim();
- Matcher m = yearMonthPattern.matcher(s);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Interval string does not match year-month format of 'y-m': " + s);
- } else {
- try {
- int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
- int years = (int) toLongWithRange("year", m.group(2), 0,
Integer.MAX_VALUE);
- int months = (int) toLongWithRange("month", m.group(3), 0, 11);
- result = new CalendarInterval(sign * (years * 12 + months), 0);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Error parsing interval year-month string: " + e.getMessage(), e);
- }
- }
- return result;
- }
-
- /**
- * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and
[-]HH:mm:ss.nnnnnnnnn
- *
- * adapted from HiveIntervalDayTime.valueOf
- */
- public static CalendarInterval fromDayTimeString(String s) throws
IllegalArgumentException {
- return fromDayTimeString(s, "day", "second");
- }
-
- /**
- * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and
[-]HH:mm:ss.nnnnnnnnn
- *
- * adapted from HiveIntervalDayTime.valueOf.
- * Below interval conversion patterns are supported:
- * - DAY TO (HOUR|MINUTE|SECOND)
- * - HOUR TO (MINUTE|SECOND)
- * - MINUTE TO SECOND
- */
- public static CalendarInterval fromDayTimeString(String s, String from,
String to)
- throws IllegalArgumentException {
- CalendarInterval result = null;
- if (s == null) {
- throw new IllegalArgumentException("Interval day-time string was null");
- }
- s = s.trim();
- Matcher m = dayTimePattern.matcher(s);
- if (!m.matches()) {
- throw new IllegalArgumentException(
- "Interval string does not match day-time format of 'd h:m:s.n': " + s);
- } else {
- try {
- int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
- long days = m.group(2) == null ? 0 : toLongWithRange("day", m.group(3),
- 0, Integer.MAX_VALUE);
- long hours = 0;
- long minutes;
- long seconds = 0;
- if (m.group(5) != null || from.equals("minute")) { // 'HH:mm:ss' or
'mm:ss minute'
- hours = toLongWithRange("hour", m.group(5), 0, 23);
- minutes = toLongWithRange("minute", m.group(6), 0, 59);
- seconds = toLongWithRange("second", m.group(7), 0, 59);
- } else if (m.group(8) != null){ // 'mm:ss.nn'
- minutes = toLongWithRange("minute", m.group(6), 0, 59);
- seconds = toLongWithRange("second", m.group(7), 0, 59);
- } else { // 'HH:mm'
- hours = toLongWithRange("hour", m.group(6), 0, 23);
- minutes = toLongWithRange("second", m.group(7), 0, 59);
- }
- // Hive allow nanosecond precision interval
- String nanoStr = m.group(9) == null ? null : (m.group(9) +
"000000000").substring(0, 9);
- long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
- switch (to) {
- case "hour":
- minutes = 0;
- seconds = 0;
- nanos = 0;
- break;
- case "minute":
- seconds = 0;
- nanos = 0;
- break;
- case "second":
- // No-op
- break;
- default:
- throw new IllegalArgumentException(
- String.format("Cannot support (interval '%s' %s to %s)
expression", s, from, to));
- }
- result = new CalendarInterval(0, sign * (
- days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes *
MICROS_PER_MINUTE +
- seconds * MICROS_PER_SECOND + nanos / 1000L));
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Error parsing interval day-time string: " + e.getMessage(), e);
- }
- }
- return result;
- }
-
- public static CalendarInterval fromUnitStrings(String[] units, String[]
values)
- throws IllegalArgumentException {
- assert units.length == values.length;
- int months = 0;
- long microseconds = 0;
-
- for (int i = 0; i < units.length; i++) {
- try {
- switch (units[i]) {
- case "year":
- months = Math.addExact(months,
Math.multiplyExact(Integer.parseInt(values[i]), 12));
- break;
- case "month":
- months = Math.addExact(months, Integer.parseInt(values[i]));
- break;
- case "week":
- microseconds = Math.addExact(
- microseconds,
- Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_WEEK));
- break;
- case "day":
- microseconds = Math.addExact(
- microseconds,
- Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_DAY));
- break;
- case "hour":
- microseconds = Math.addExact(
- microseconds,
- Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_HOUR));
- break;
- case "minute":
- microseconds = Math.addExact(
- microseconds,
- Math.multiplyExact(Long.parseLong(values[i]),
MICROS_PER_MINUTE));
- break;
- case "second": {
- microseconds = Math.addExact(microseconds,
parseSecondNano(values[i]));
- break;
- }
- case "millisecond":
- microseconds = Math.addExact(
- microseconds,
- Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_MILLI));
- break;
- case "microsecond":
- microseconds = Math.addExact(microseconds,
Long.parseLong(values[i]));
- break;
- }
- } catch (Exception e) {
- throw new IllegalArgumentException("Error parsing interval string: " +
e.getMessage(), e);
- }
- }
- return new CalendarInterval(months, microseconds);
- }
-
- /**
- * Parse second_nano string in ss.nnnnnnnnn format to microseconds
- */
- public static long parseSecondNano(String secondNano) throws
IllegalArgumentException {
- String[] parts = secondNano.split("\\.");
- if (parts.length == 1) {
- return toLongWithRange("second", parts[0], Long.MIN_VALUE /
MICROS_PER_SECOND,
- Long.MAX_VALUE / MICROS_PER_SECOND) * MICROS_PER_SECOND;
-
- } else if (parts.length == 2) {
- long seconds = parts[0].equals("") ? 0L : toLongWithRange("second",
parts[0],
- Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE /
MICROS_PER_SECOND);
- long nanos = toLongWithRange("nanosecond", parts[1], 0L, 999999999L);
- return seconds * MICROS_PER_SECOND + nanos / 1000L;
-
- } else {
- throw new IllegalArgumentException(
- "Interval string does not match second-nano format of ss.nnnnnnnnn");
- }
- }
-
public final int months;
public final long microseconds;
diff --git
a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
index 9f3262b..5e418c1 100644
---
a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
+++
b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
@@ -61,72 +61,6 @@ public class CalendarIntervalSuite {
}
@Test
- public void fromYearMonthStringTest() {
- String input;
- CalendarInterval i;
-
- input = "99-10";
- i = new CalendarInterval(99 * 12 + 10, 0L);
- assertEquals(fromYearMonthString(input), i);
-
- input = "-8-10";
- i = new CalendarInterval(-8 * 12 - 10, 0L);
- assertEquals(fromYearMonthString(input), i);
-
- try {
- input = "99-15";
- fromYearMonthString(input);
- fail("Expected to throw an exception for the invalid input");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("month 15 outside range"));
- }
- }
-
- @Test
- public void fromDayTimeStringTest() {
- String input;
- CalendarInterval i;
-
- input = "5 12:40:30.999999999";
- i = new CalendarInterval(0, 5 * MICROS_PER_DAY + 12 * MICROS_PER_HOUR +
- 40 * MICROS_PER_MINUTE + 30 * MICROS_PER_SECOND + 999999L);
- assertEquals(fromDayTimeString(input), i);
-
- input = "10 0:12:0.888";
- i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
- 888 * MICROS_PER_MILLI);
- assertEquals(fromDayTimeString(input), i);
-
- input = "-3 0:0:0";
- i = new CalendarInterval(0, -3 * MICROS_PER_DAY);
- assertEquals(fromDayTimeString(input), i);
-
- try {
- input = "5 30:12:20";
- fromDayTimeString(input);
- fail("Expected to throw an exception for the invalid input");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("hour 30 outside range"));
- }
-
- try {
- input = "5 30-12";
- fromDayTimeString(input);
- fail("Expected to throw an exception for the invalid input");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("not match day-time format"));
- }
-
- try {
- input = "5 1:12:20";
- fromDayTimeString(input, "hour", "microsecond");
- fail("Expected to throw an exception for the invalid convention type");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("Cannot support (interval"));
- }
- }
-
- @Test
public void addTest() {
CalendarInterval input1 = new CalendarInterval(3, 1 * MICROS_PER_HOUR);
CalendarInterval input2 = new CalendarInterval(2, 100 * MICROS_PER_HOUR);
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 893003d..072b9a1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -108,7 +108,7 @@ class AstBuilder(conf: SQLConf) extends
SqlBaseBaseVisitor[AnyRef] with Logging
}.toArray
val values = ctx.intervalValue().asScala.map(getIntervalValue).toArray
try {
- CalendarInterval.fromUnitStrings(units, values)
+ IntervalUtils.fromUnitStrings(units, values)
} catch {
case i: IllegalArgumentException =>
val e = new ParseException(i.getMessage, ctx)
@@ -1953,21 +1953,21 @@ class AstBuilder(conf: SQLConf) extends
SqlBaseBaseVisitor[AnyRef] with Logging
val unitText = unit.getText.toLowerCase(Locale.ROOT)
val interval = (unitText,
Option(to).map(_.getText.toLowerCase(Locale.ROOT))) match {
case (u, None) =>
- CalendarInterval.fromUnitStrings(Array(normalizeInternalUnit(u)),
Array(s))
+ IntervalUtils.fromUnitStrings(Array(normalizeInternalUnit(u)),
Array(s))
case ("year", Some("month")) =>
- CalendarInterval.fromYearMonthString(s)
+ IntervalUtils.fromYearMonthString(s)
case ("day", Some("hour")) =>
- CalendarInterval.fromDayTimeString(s, "day", "hour")
+ IntervalUtils.fromDayTimeString(s, "day", "hour")
case ("day", Some("minute")) =>
- CalendarInterval.fromDayTimeString(s, "day", "minute")
+ IntervalUtils.fromDayTimeString(s, "day", "minute")
case ("day", Some("second")) =>
- CalendarInterval.fromDayTimeString(s, "day", "second")
+ IntervalUtils.fromDayTimeString(s, "day", "second")
case ("hour", Some("minute")) =>
- CalendarInterval.fromDayTimeString(s, "hour", "minute")
+ IntervalUtils.fromDayTimeString(s, "hour", "minute")
case ("hour", Some("second")) =>
- CalendarInterval.fromDayTimeString(s, "hour", "second")
+ IntervalUtils.fromDayTimeString(s, "hour", "second")
case ("minute", Some("second")) =>
- CalendarInterval.fromDayTimeString(s, "minute", "second")
+ IntervalUtils.fromDayTimeString(s, "minute", "second")
case (from, Some(t)) =>
throw new ParseException(s"Intervals FROM $from TO $t are not
supported.", ctx)
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 14fd153..f55b054 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -17,6 +17,10 @@
package org.apache.spark.sql.catalyst.util
+import java.util.regex.Pattern
+
+import scala.util.control.NonFatal
+
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
import org.apache.spark.sql.types.Decimal
import org.apache.spark.unsafe.types.CalendarInterval
@@ -119,4 +123,198 @@ object IntervalUtils {
case _: IllegalArgumentException => null
}
}
+
+ private def toLongWithRange(
+ fieldName: String,
+ s: String,
+ minValue: Long,
+ maxValue: Long): Long = {
+ val result = if (s == null) 0L else s.toLong
+ require(minValue <= result && result <= maxValue,
+ s"$fieldName $result outside range [$minValue, $maxValue]")
+
+ result
+ }
+
+ private val yearMonthPattern = "^([+|-])?(\\d+)-(\\d+)$".r
+
+ /**
+ * Parse YearMonth string in form: [+|-]YYYY-MM
+ *
+ * adapted from HiveIntervalYearMonth.valueOf
+ */
+ def fromYearMonthString(input: String): CalendarInterval = {
+ require(input != null, "Interval year-month string must be not null")
+ def toInterval(yearStr: String, monthStr: String): CalendarInterval = {
+ try {
+ val years = toLongWithRange("year", yearStr, 0,
Integer.MAX_VALUE).toInt
+ val months = toLongWithRange("month", monthStr, 0, 11).toInt
+ val totalMonths = Math.addExact(Math.multiplyExact(years, 12), months)
+ new CalendarInterval(totalMonths, 0)
+ } catch {
+ case NonFatal(e) =>
+ throw new IllegalArgumentException(
+ s"Error parsing interval year-month string: ${e.getMessage}", e)
+ }
+ }
+ assert(input.length == input.trim.length)
+ input match {
+ case yearMonthPattern("-", yearStr, monthStr) =>
+ toInterval(yearStr, monthStr).negate()
+ case yearMonthPattern(_, yearStr, monthStr) =>
+ toInterval(yearStr, monthStr)
+ case _ =>
+ throw new IllegalArgumentException(
+ s"Interval string does not match year-month format of 'y-m': $input")
+ }
+ }
+
+ /**
+ * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and
[-]HH:mm:ss.nnnnnnnnn
+ *
+ * adapted from HiveIntervalDayTime.valueOf
+ */
+ def fromDayTimeString(s: String): CalendarInterval = {
+ fromDayTimeString(s, "day", "second")
+ }
+
+ private val dayTimePattern =
+ "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r
+
+ /**
+ * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and
[-]HH:mm:ss.nnnnnnnnn
+ *
+ * adapted from HiveIntervalDayTime.valueOf.
+ * Below interval conversion patterns are supported:
+ * - DAY TO (HOUR|MINUTE|SECOND)
+ * - HOUR TO (MINUTE|SECOND)
+ * - MINUTE TO SECOND
+ */
+ def fromDayTimeString(input: String, from: String, to: String):
CalendarInterval = {
+ require(input != null, "Interval day-time string must be not null")
+ assert(input.length == input.trim.length)
+ val m = dayTimePattern.pattern.matcher(input)
+ require(m.matches, s"Interval string must match day-time format of 'd
h:m:s.n': $input")
+
+ try {
+ val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1
+ val days = if (m.group(2) == null) {
+ 0
+ } else {
+ toLongWithRange("day", m.group(3), 0, Integer.MAX_VALUE)
+ }
+ var hours: Long = 0L
+ var minutes: Long = 0L
+ var seconds: Long = 0L
+ if (m.group(5) != null || from == "minute") { // 'HH:mm:ss' or 'mm:ss
minute'
+ hours = toLongWithRange("hour", m.group(5), 0, 23)
+ minutes = toLongWithRange("minute", m.group(6), 0, 59)
+ seconds = toLongWithRange("second", m.group(7), 0, 59)
+ } else if (m.group(8) != null) { // 'mm:ss.nn'
+ minutes = toLongWithRange("minute", m.group(6), 0, 59)
+ seconds = toLongWithRange("second", m.group(7), 0, 59)
+ } else { // 'HH:mm'
+ hours = toLongWithRange("hour", m.group(6), 0, 23)
+ minutes = toLongWithRange("second", m.group(7), 0, 59)
+ }
+ // Hive allow nanosecond precision interval
+ val nanoStr = if (m.group(9) == null) {
+ null
+ } else {
+ (m.group(9) + "000000000").substring(0, 9)
+ }
+ var nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L)
+ to match {
+ case "hour" =>
+ minutes = 0
+ seconds = 0
+ nanos = 0
+ case "minute" =>
+ seconds = 0
+ nanos = 0
+ case "second" =>
+ // No-op
+ case _ =>
+ throw new IllegalArgumentException(
+ s"Cannot support (interval '$input' $from to $to) expression")
+ }
+ var micros = nanos / DateTimeUtils.NANOS_PER_MICROS
+ micros = Math.addExact(micros, Math.multiplyExact(days,
DateTimeUtils.MICROS_PER_DAY))
+ micros = Math.addExact(micros, Math.multiplyExact(hours,
MICROS_PER_HOUR))
+ micros = Math.addExact(micros, Math.multiplyExact(minutes,
MICROS_PER_MINUTE))
+ micros = Math.addExact(micros, Math.multiplyExact(seconds,
DateTimeUtils.MICROS_PER_SECOND))
+ new CalendarInterval(0, sign * micros)
+ } catch {
+ case e: Exception =>
+ throw new IllegalArgumentException(
+ s"Error parsing interval day-time string: ${e.getMessage}", e)
+ }
+ }
+
+ def fromUnitStrings(units: Array[String], values: Array[String]):
CalendarInterval = {
+ assert(units.length == values.length)
+ var months: Int = 0
+ var microseconds: Long = 0
+ var i = 0
+ while (i < units.length) {
+ try {
+ units(i) match {
+ case "year" =>
+ months = Math.addExact(months, Math.multiplyExact(values(i).toInt,
12))
+ case "month" =>
+ months = Math.addExact(months, values(i).toInt)
+ case "week" =>
+ val weeksUs = Math.multiplyExact(values(i).toLong, 7 *
DateTimeUtils.MICROS_PER_DAY)
+ microseconds = Math.addExact(microseconds, weeksUs)
+ case "day" =>
+ val daysUs = Math.multiplyExact(values(i).toLong,
DateTimeUtils.MICROS_PER_DAY)
+ microseconds = Math.addExact(microseconds, daysUs)
+ case "hour" =>
+ val hoursUs = Math.multiplyExact(values(i).toLong, MICROS_PER_HOUR)
+ microseconds = Math.addExact(microseconds, hoursUs)
+ case "minute" =>
+ val minutesUs = Math.multiplyExact(values(i).toLong,
MICROS_PER_MINUTE)
+ microseconds = Math.addExact(microseconds, minutesUs)
+ case "second" =>
+ microseconds = Math.addExact(microseconds,
parseSecondNano(values(i)))
+ case "millisecond" =>
+ val millisUs = Math.multiplyExact(values(i).toLong,
DateTimeUtils.MICROS_PER_MILLIS)
+ microseconds = Math.addExact(microseconds, millisUs)
+ case "microsecond" =>
+ microseconds = Math.addExact(microseconds, values(i).toLong)
+ }
+ } catch {
+ case e: Exception =>
+ throw new IllegalArgumentException(s"Error parsing interval string:
${e.getMessage}", e)
+ }
+ i += 1
+ }
+ new CalendarInterval(months, microseconds)
+ }
+
+ /**
+ * Parse second_nano string in ss.nnnnnnnnn format to microseconds
+ */
+ private def parseSecondNano(secondNano: String): Long = {
+ def parseSeconds(secondsStr: String): Long = {
+ toLongWithRange(
+ "second",
+ secondsStr,
+ Long.MinValue / DateTimeUtils.MICROS_PER_SECOND,
+ Long.MaxValue / DateTimeUtils.MICROS_PER_SECOND) *
DateTimeUtils.MICROS_PER_SECOND
+ }
+ def parseNanos(nanosStr: String): Long = {
+ toLongWithRange("nanosecond", nanosStr, 0L, 999999999L) /
DateTimeUtils.NANOS_PER_MICROS
+ }
+
+ secondNano.split("\\.") match {
+ case Array(secondsStr) => parseSeconds(secondsStr)
+ case Array("", nanosStr) => parseNanos(nanosStr)
+ case Array(secondsStr, nanosStr) =>
+ Math.addExact(parseSeconds(secondsStr), parseNanos(nanosStr))
+ case _ =>
+ throw new IllegalArgumentException(
+ "Interval string does not match second-nano format of ss.nnnnnnnnn")
+ }
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index e10aa60..74d1606 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -810,7 +810,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper
checkEvaluation(new Sequence(
Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
Literal(Timestamp.valueOf("2023-01-01 00:00:00")),
- Literal(CalendarInterval.fromYearMonthString("1-5"))),
+ Literal(IntervalUtils.fromYearMonthString("1-5"))),
Seq(
Timestamp.valueOf("2018-01-01 00:00:00.000"),
Timestamp.valueOf("2019-06-01 00:00:00.000"),
@@ -820,7 +820,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper
checkEvaluation(new Sequence(
Literal(Timestamp.valueOf("2022-04-01 00:00:00")),
Literal(Timestamp.valueOf("2017-01-01 00:00:00")),
- Literal(CalendarInterval.fromYearMonthString("1-5").negate())),
+ Literal(IntervalUtils.fromYearMonthString("1-5").negate())),
Seq(
Timestamp.valueOf("2022-04-01 00:00:00.000"),
Timestamp.valueOf("2020-11-01 00:00:00.000"),
@@ -894,7 +894,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper
checkEvaluation(new Sequence(
Literal(Date.valueOf("2018-01-01")),
Literal(Date.valueOf("2023-01-01")),
- Literal(CalendarInterval.fromYearMonthString("1-5"))),
+ Literal(IntervalUtils.fromYearMonthString("1-5"))),
Seq(
Date.valueOf("2018-01-01"),
Date.valueOf("2019-06-01"),
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 86b3aa81..5a7b3ff 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -597,7 +597,7 @@ class ExpressionParserSuite extends AnalysisTest {
"microsecond")
def intervalLiteral(u: String, s: String): Literal = {
- Literal(CalendarInterval.fromUnitStrings(Array(u), Array(s)))
+ Literal(IntervalUtils.fromUnitStrings(Array(u), Array(s)))
}
test("intervals") {
@@ -637,7 +637,7 @@ class ExpressionParserSuite extends AnalysisTest {
// Year-Month intervals.
val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0")
yearMonthValues.foreach { value =>
- val result = Literal(CalendarInterval.fromYearMonthString(value))
+ val result = Literal(IntervalUtils.fromYearMonthString(value))
checkIntervals(s"'$value' year to month", result)
}
@@ -650,7 +650,7 @@ class ExpressionParserSuite extends AnalysisTest {
"-1 0:0:0",
"1 0:0:1")
datTimeValues.foreach { value =>
- val result = Literal(CalendarInterval.fromDayTimeString(value))
+ val result = Literal(IntervalUtils.fromDayTimeString(value))
checkIntervals(s"'$value' day to second", result)
}
@@ -662,7 +662,7 @@ class ExpressionParserSuite extends AnalysisTest {
"0:0:0",
"0:0:1")
hourTimeValues.foreach { value =>
- val result = Literal(CalendarInterval.fromDayTimeString(value))
+ val result = Literal(IntervalUtils.fromDayTimeString(value))
checkIntervals(s"'$value' hour to second", result)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index e48779a..9addc39 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.util
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.util.IntervalUtils.fromString
+import org.apache.spark.sql.catalyst.util.IntervalUtils.{fromDayTimeString,
fromString, fromYearMonthString}
import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.spark.unsafe.types.CalendarInterval._
@@ -87,4 +87,65 @@ class IntervalUtilsSuite extends SparkFunSuite {
assert(fromString(input2) == result)
}
}
+
+ test("from year-month string") {
+ assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10,
0L))
+ assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 +
10, 0L))
+ assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10,
0L))
+
+ try {
+ fromYearMonthString("99-15")
+ fail("Expected to throw an exception for the invalid input")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("month 15 outside range"))
+ }
+
+ try {
+ fromYearMonthString("9a9-15")
+ fail("Expected to throw an exception for the invalid input")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("Interval string does not match
year-month format"))
+ }
+ }
+
+ test("from day-time string") {
+ assert(fromDayTimeString("5 12:40:30.999999999") ===
+ new CalendarInterval(
+ 0,
+ 5 * MICROS_PER_DAY +
+ 12 * MICROS_PER_HOUR +
+ 40 * MICROS_PER_MINUTE +
+ 30 * MICROS_PER_SECOND + 999999L))
+ assert(fromDayTimeString("10 0:12:0.888") ===
+ new CalendarInterval(
+ 0,
+ 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE + 888 * MICROS_PER_MILLI))
+ assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3 *
MICROS_PER_DAY))
+
+ try {
+ fromDayTimeString("5 30:12:20")
+ fail("Expected to throw an exception for the invalid input")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("hour 30 outside range"))
+ }
+
+ try {
+ fromDayTimeString("5 30-12")
+ fail("Expected to throw an exception for the invalid input")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("must match day-time format"))
+ }
+
+ try {
+ fromDayTimeString("5 1:12:20", "hour", "microsecond")
+ fail("Expected to throw an exception for the invalid convention type")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("Cannot support (interval"))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
