This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new fe18354 [SPARK-35679][SQL] instantToMicros overflow fe18354 is described below commit fe1835406ac46eca2b147e59a92a91ba04f61121 Author: dgd-contributor <dgd_contribu...@viettel.com.vn> AuthorDate: Thu Jun 10 08:08:51 2021 +0300 [SPARK-35679][SQL] instantToMicros overflow With Long.minValue cast to an instant, secs will be floored in function microsToInstant and cause overflow when multiply with Micros_per_second ``` def microsToInstant(micros: Long): Instant = { val secs = Math.floorDiv(micros, MICROS_PER_SECOND) // Unfolded Math.floorMod(us, MICROS_PER_SECOND) to reuse the result of // the above calculation of `secs` via `floorDiv`. val mos = micros - secs * MICROS_PER_SECOND <- it will overflow here Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS) } ``` But the overflow is acceptable because it won't produce any change to the result However, when convert the instant back to micro value, it will raise Overflow Error ``` def instantToMicros(instant: Instant): Long = { val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) <- It overflow here val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) result } ``` Code to reproduce this error ``` instantToMicros(microToInstant(Long.MinValue)) ``` No Test added Closes #32839 from dgd-contributor/SPARK-35679_instantToMicro. Authored-by: dgd-contributor <dgd_contribu...@viettel.com.vn> Signed-off-by: Max Gekk <max.g...@gmail.com> (cherry picked from commit aa3de4077302fe7e0b23b01a338c7feab0e5974e) Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../spark/sql/catalyst/util/DateTimeUtils.scala | 20 +++++++++++++++++--- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 5 +++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 52df227..2e242f2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -413,10 +413,24 @@ object DateTimeUtils { } } + // See issue SPARK-35679 + // min second cause overflow in instant to micro + private val MIN_SECONDS = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND) + + /** + * Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given + * instance of `java.time.Instant`. The epoch microsecond count is a simple incrementing count of + * microseconds where microsecond 0 is 1970-01-01 00:00:00Z. + */ def instantToMicros(instant: Instant): Long = { - val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) - val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) - result + val secs = instant.getEpochSecond + if (secs == MIN_SECONDS) { + val us = Math.multiplyExact(secs + 1, MICROS_PER_SECOND) + Math.addExact(us, NANOSECONDS.toMicros(instant.getNano) - MICROS_PER_SECOND) + } else { + val us = Math.multiplyExact(secs, MICROS_PER_SECOND) + Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) + } } def microsToInstant(us: Long): Instant = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index d457c88..d451aff 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -686,4 +686,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { assert(toDate("tomorrow CET ", zoneId).get === today + 1) } } + + test("SPARK-35679: instantToMicros should be able to return microseconds of Long.MinValue") { + assert(instantToMicros(microsToInstant(Long.MaxValue)) === Long.MaxValue) + assert(instantToMicros(microsToInstant(Long.MinValue)) === Long.MinValue) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org