This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1cd649d4b076 [SPARK-57850][SQL] Return the correct fractional-second
scale from EXTRACT(SECOND) on TIME
1cd649d4b076 is described below
commit 1cd649d4b076d91d820258387a6bcf0d878021b0
Author: Anupam Yadav <[email protected]>
AuthorDate: Mon Jul 13 23:40:24 2026 +0200
[SPARK-57850][SQL] Return the correct fractional-second scale from
EXTRACT(SECOND) on TIME
### What changes were proposed in this pull request?
`EXTRACT(SECOND FROM t)` / `date_part('SECOND', t)` on a `TIME(p)` value
returned a hardcoded `DecimalType(8, 6)` regardless of the TIME precision. This
changes the result type to `DecimalType(2 + p, p)`, so the fractional-second
scale matches the value's precision (e.g. `TIME(0)` -> `decimal(2,0)`,
`TIME(6)` -> `decimal(8,6)`).
### Why are the changes needed?
The scale was always 6, so `EXTRACT(SECOND)` on e.g. `TIME(0)` reported an
incorrect `decimal(8,6)` type with spurious fractional digits. The scale should
reflect the value's declared precision.
### Does this PR introduce _any_ user-facing change?
Yes. `EXTRACT(SECOND FROM <TIME(p)>)` / `date_part('SECOND', <TIME(p)>)`
now returns `DecimalType(2 + p, p)` instead of always `DecimalType(8, 6)`.
TIMESTAMP behavior is unchanged.
### How was this patch tested?
Unit tests in `TimeExpressionsSuite` asserting both the result
`DecimalType` and the value across TIME precisions 0-6; the `time.sql`
`SQLQueryTestSuite` golden was updated accordingly.
### Was this patch authored or co-authored using generative AI tooling?
Authored with assistance by Claude Opus 4.8.
Closes #56931 from yadavay-amzn/SPARK-57850.
Authored-by: Anupam Yadav <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../catalyst/expressions/datetimeExpressions.scala | 7 ++++---
.../sql/catalyst/expressions/timeExpressions.scala | 2 +-
.../spark/sql/catalyst/util/DateTimeUtils.scala | 2 +-
.../expressions/TimeExpressionsSuite.scala | 19 ++++++++++++++---
.../test/resources/sql-tests/results/time.sql.out | 24 +++++++++++-----------
5 files changed, 34 insertions(+), 20 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index e7c6a1091861..3e49a0fca048 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -477,9 +477,10 @@ case class SecondWithFraction(child: Expression,
timeZoneId: Option[String] = No
* This is the nanosecond counterpart of [[SecondWithFraction]] on the
* `EXTRACT(SECOND FROM ...)` / `date_part('SECOND', ...)` path. Unlike the
integer time-of-day
* fields, the result depends on the sub-microsecond digits, so the input is
not cast down to
- * microseconds. The scale is fixed at the maximum nanosecond precision, like
- * [[SecondsOfTimeWithFraction]] does for TIME values: digits below the type's
precision `p` are
- * always zero because values are floored to `p` at the type boundary.
+ * microseconds. The scale is fixed at the maximum nanosecond precision
(`DECIMAL(11, 9)`): digits
+ * below the type's precision `p` are always zero because values are floored
to `p` at the type
+ * boundary. This differs from `EXTRACT(SECOND)` on the TIME type
([[SecondsOfTimeWithFraction]]),
+ * which uses a per-precision `DECIMAL(2 + p, p)` scale.
*/
case class SecondWithFractionNanos(child: Expression, timeZoneId:
Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
index 1c9d6b335e8d..2081007d3e50 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
@@ -371,7 +371,7 @@ case class SecondsOfTimeWithFraction(child: Expression)
}
StaticInvoke(
classOf[DateTimeUtils.type],
- DecimalType(8, 6),
+ DecimalType(2 + precision, precision),
"getSecondsOfTimeWithFraction",
Seq(child, Literal(precision)),
Seq(child.dataType, IntegerType))
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index be2cc6098e73..b1bf6526c181 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -175,7 +175,7 @@ object DateTimeUtils extends SparkDateTimeUtils {
val scaleFactor = math.pow(10, precision).toLong
val scaledFraction = (nanos % NANOS_PER_SECOND) * scaleFactor /
NANOS_PER_SECOND
val fraction = scaledFraction.toDouble / scaleFactor
- Decimal(seconds + fraction, 8, 6)
+ Decimal(seconds + fraction, 2 + precision, precision)
}
/**
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
index 6db6115a1e5e..e03d8cb39ea2 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
@@ -360,9 +360,22 @@ class TimeExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
4 -> 15.9876,
5 -> 15.98765,
6 -> 15.987654).foreach { case (precision, expected) =>
- checkEvaluation(
- SecondsOfTimeWithFraction(Literal(localTime(13, 11, 15, 987654),
TimeType(precision))),
- BigDecimal(expected))
+ val expr = SecondsOfTimeWithFraction(
+ Literal(localTime(13, 11, 15, 987654), TimeType(precision)))
+ assert(expr.dataType == DecimalType(2 + precision, precision),
+ s"TIME($precision) SECOND should have DecimalType(${2 + precision},
$precision)")
+ checkEvaluation(expr, BigDecimal(expected))
+ }
+ // Precisions 7-9 require sub-microsecond nanos
+ Seq(
+ 7 -> BigDecimal("15.9876543"),
+ 8 -> BigDecimal("15.98765432"),
+ 9 -> BigDecimal("15.987654321")).foreach { case (precision, expected) =>
+ val expr = SecondsOfTimeWithFraction(
+ Literal(localTime(13, 11, 15, 987654, 321), TimeType(precision)))
+ assert(expr.dataType == DecimalType(2 + precision, precision),
+ s"TIME($precision) SECOND should have DecimalType(${2 + precision},
$precision)")
+ checkEvaluation(expr, expected)
}
// Verify NULL handling
checkEvaluation(
diff --git a/sql/core/src/test/resources/sql-tests/results/time.sql.out
b/sql/core/src/test/resources/sql-tests/results/time.sql.out
index dace78ab938d..31444a600b49 100644
--- a/sql/core/src/test/resources/sql-tests/results/time.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/time.sql.out
@@ -463,49 +463,49 @@ struct<extract(SECS FROM TIME
'23:59:58.987654'):decimal(8,6)>
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(0)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(0))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(0))):decimal(2,0)>
-- !query output
-1.000000
+1
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(1)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(1))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(1))):decimal(3,1)>
-- !query output
-1.900000
+1.9
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(2)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(2))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(2))):decimal(4,2)>
-- !query output
-1.980000
+1.98
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(3)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(3))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(3))):decimal(5,3)>
-- !query output
-1.987000
+1.987
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(4)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(4))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(4))):decimal(6,4)>
-- !query output
-1.987600
+1.9876
-- !query
select extract(SECOND FROM cast('09:08:01.987654' as time(5)))
-- !query schema
-struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(5))):decimal(8,6)>
+struct<extract(SECOND FROM CAST(09:08:01.987654 AS TIME(5))):decimal(7,5)>
-- !query output
-1.987650
+1.98765
-- !query
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
