This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 80f7989 [SPARK-35734][SQL] Format day-time intervals using type fields
80f7989 is described below
commit 80f7989d9a761eff1a0b3c64ec3aabb81506953d
Author: Kousuke Saruta <saru...@oss.nttdata.com>
AuthorDate: Sat Jun 12 21:45:12 2021 +0300
[SPARK-35734][SQL] Format day-time intervals using type fields
### What changes were proposed in this pull request?
This PR add a feature which formats day-time interval to strings using the
start and end fields of `DayTimeIntervalType`.
### Why are the changes needed?
Currently, they are ignored, and any `DayTimeIntervalType` is formatted as
`INTERVAL DAY TO SECOND.`
### Does this PR introduce _any_ user-facing change?
Yes. The format of day-time intervals is determined the start and end
fields.
### How was this patch tested?
New test.
Closes #32891 from sarutak/interval-format.
Authored-by: Kousuke Saruta <saru...@oss.nttdata.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../spark/sql/catalyst/util/IntervalUtils.scala | 58 +++++++++++++--
.../sql/catalyst/util/IntervalUtilsSuite.scala | 84 ++++++++++++++++++++++
2 files changed, 138 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index c18cca9..dda5581 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -21,6 +21,7 @@ import java.time.{Duration, Period}
import java.time.temporal.ChronoUnit
import java.util.concurrent.TimeUnit
+import scala.collection.mutable
import scala.util.control.NonFatal
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -28,7 +29,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE,
HIVE_STYLE, IntervalStyle}
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.Decimal
+import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
// The style of textual representation of intervals
@@ -960,18 +961,34 @@ object IntervalUtils {
def toDayTimeIntervalString(
micros: Long,
style: IntervalStyle,
- // TODO(SPARK-35734): Format day-time intervals using type fields
startField: Byte,
endField: Byte): String = {
var sign = ""
var rest = micros
+ val from = DayTimeIntervalType.fieldToString(startField).toUpperCase
+ val to = DayTimeIntervalType.fieldToString(endField).toUpperCase
if (micros < 0) {
if (micros == Long.MinValue) {
// Especial handling of minimum `Long` value because negate op
overflows `Long`.
// seconds = 106751991 * (24 * 60 * 60) + 4 * 60 * 60 + 54 =
9223372036854
// microseconds = -9223372036854000000L-775808 == Long.MinValue
val minIntervalString = style match {
- case ANSI_STYLE => "INTERVAL '-106751991 04:00:54.775808' DAY TO
SECOND"
+ case ANSI_STYLE =>
+ val baseStr = "-106751991 04:00:54.775808"
+ val fromPos = startField match {
+ case DayTimeIntervalType.DAY => 0
+ case DayTimeIntervalType.HOUR => 11
+ case DayTimeIntervalType.MINUTE => 14
+ case DayTimeIntervalType.SECOND => 17
+ }
+ val toPos = endField match {
+ case DayTimeIntervalType.DAY => 10
+ case DayTimeIntervalType.HOUR => 13
+ case DayTimeIntervalType.MINUTE => 16
+ case DayTimeIntervalType.SECOND => baseStr.length
+ }
+ val postfix = if (startField == endField) from else s"$from TO $to"
+ s"INTERVAL '${baseStr.substring(fromPos, toPos)}' $postfix"
case HIVE_STYLE => "-106751991 04:00:54.775808000"
}
return minIntervalString
@@ -992,7 +1009,40 @@ object IntervalUtils {
val secStr = java.math.BigDecimal.valueOf(secondsWithFraction, 6)
.stripTrailingZeros()
.toPlainString()
- f"INTERVAL '$sign$days $hours%02d:$minutes%02d:$leadSecZero$secStr'
DAY TO SECOND"
+ val formatBuilder = new StringBuilder("INTERVAL '")
+ if (startField == endField) {
+ startField match {
+ case DayTimeIntervalType.DAY => formatBuilder.append(s"$sign$days'
")
+ case DayTimeIntervalType.HOUR =>
formatBuilder.append(f"$hours%02d' ")
+ case DayTimeIntervalType.MINUTE =>
formatBuilder.append(f"$minutes%02d' ")
+ case DayTimeIntervalType.SECOND =>
formatBuilder.append(s"$leadSecZero$secStr' ")
+ }
+ formatBuilder.append(from).toString
+ } else {
+ val formatArgs = new mutable.ArrayBuffer[Long]
+ if (startField <= DayTimeIntervalType.DAY && DayTimeIntervalType.DAY
< endField) {
+ formatBuilder.append(s"$sign$days ")
+ }
+ if (startField <= DayTimeIntervalType.HOUR &&
DayTimeIntervalType.HOUR < endField) {
+ formatBuilder.append("%02d:")
+ formatArgs.append(hours)
+ }
+ if (startField <= DayTimeIntervalType.MINUTE &&
DayTimeIntervalType.MINUTE < endField) {
+ formatBuilder.append("%02d:")
+ formatArgs.append(minutes)
+ }
+ endField match {
+ case DayTimeIntervalType.HOUR =>
+ formatBuilder.append("%02d' ")
+ formatArgs.append(hours)
+ case DayTimeIntervalType.MINUTE =>
+ formatBuilder.append("%02d' ")
+ formatArgs.append(minutes)
+ case DayTimeIntervalType.SECOND =>
+ formatBuilder.append(s"$leadSecZero$secStr' ")
+ }
+ formatBuilder.append(s"$from TO
$to").toString.format(formatArgs.toSeq: _*)
+ }
case HIVE_STYLE =>
val seconds = secondsWithFraction / MICROS_PER_SECOND
val nanos = (secondsWithFraction % MICROS_PER_SECOND) *
NANOS_PER_MICROS
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 8db63e7..0d496c9 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -534,4 +534,88 @@ class IntervalUtilsSuite extends SparkFunSuite with
SQLHelper {
assert(toDayTimeIntervalString(micros, HIVE_STYLE, DAY, SECOND) ===
hiveIntervalStr)
}
}
+
+ test("SPARK-35734: Format day-time intervals using type fields") {
+ import DayTimeIntervalType._
+ Seq(
+ 0L ->
+ ("INTERVAL '0 00:00:00' DAY TO SECOND",
+ "INTERVAL '0 00:00' DAY TO MINUTE",
+ "INTERVAL '0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00' MINUTE TO SECOND",
+ "INTERVAL '0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00' SECOND"),
+ -1L ->
+ ("INTERVAL '-0 00:00:00.000001' DAY TO SECOND",
+ "INTERVAL '-0 00:00' DAY TO MINUTE",
+ "INTERVAL '-0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00.000001' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00.000001' MINUTE TO SECOND",
+ "INTERVAL '-0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00.000001' SECOND"),
+ 10 * MICROS_PER_MILLIS ->
+ ("INTERVAL '0 00:00:00.01' DAY TO SECOND",
+ "INTERVAL '0 00:00' DAY TO MINUTE",
+ "INTERVAL '0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00.01' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00.01' MINUTE TO SECOND",
+ "INTERVAL '0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00.01' SECOND"),
+ (-123 * MICROS_PER_DAY - 3 * MICROS_PER_SECOND) ->
+ ("INTERVAL '-123 00:00:03' DAY TO SECOND",
+ "INTERVAL '-123 00:00' DAY TO MINUTE",
+ "INTERVAL '-123 00' DAY TO HOUR",
+ "INTERVAL '00:00:03' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:03' MINUTE TO SECOND",
+ "INTERVAL '-123' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '03' SECOND"),
+ Long.MinValue ->
+ ("INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND",
+ "INTERVAL '-106751991 04:00' DAY TO MINUTE",
+ "INTERVAL '-106751991 04' DAY TO HOUR",
+ "INTERVAL '04:00:54.775808' HOUR TO SECOND",
+ "INTERVAL '04:00' HOUR TO MINUTE",
+ "INTERVAL '00:54.775808' MINUTE TO SECOND",
+ "INTERVAL '-106751991' DAY",
+ "INTERVAL '04' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '54.775808' SECOND")
+ ).foreach {
+ case (
+ micros, (
+ dayToSec,
+ dayToMinute,
+ dayToHour,
+ hourToSec,
+ hourToMinute,
+ minuteToSec,
+ day,
+ hour,
+ minute,
+ sec)) =>
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, SECOND) ===
dayToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, MINUTE) ===
dayToMinute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, HOUR) ===
dayToHour)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, SECOND) ===
hourToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, MINUTE) ===
hourToMinute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, MINUTE, SECOND) ===
minuteToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, DAY) === day)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, HOUR) ===
hour)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, MINUTE, MINUTE) ===
minute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, SECOND, SECOND) ===
sec)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
