This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new 17fc3632f23 [SPARK-44457][CONNECT][TESTS] Add `truncatedTo(ChronoUnit.MICROS)` to make `ArrowEncoderSuite` in Java 17 daily test GA task pass 17fc3632f23 is described below commit 17fc3632f2344101f8318457e3f9d5f133913997 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Wed Jul 26 19:17:40 2023 -0500 [SPARK-44457][CONNECT][TESTS] Add `truncatedTo(ChronoUnit.MICROS)` to make `ArrowEncoderSuite` in Java 17 daily test GA task pass ### What changes were proposed in this pull request? Similar to SPARK-42770 | https://github.com/apache/spark/pull/40395, this pr call `truncatedTo(ChronoUnit.MICROS)` on `Instant.now()` and `LocalDateTime.now()` to ensure microsecond accuracy is used in any environment. ### Why are the changes needed? Make Java 17 daily test GA task run successfully. The Java 17 daily test GA task failed now: https://github.com/apache/spark/actions/runs/5570003581/jobs/10173767006 ``` [info] - nullable fields *** FAILED *** (169 milliseconds) [info] NullableData(null, JANUARY, E1, null, 1.000000000000000000, 2.000000000000000000, null, 4, PT0S, null, 2023-07-16, 2023-07-16, null, 2023-07-16T23:01:54.059339Z, 2023-07-16T23:01:54.059359) did not equal NullableData(null, JANUARY, E1, null, 1.000000000000000000, 2.000000000000000000, null, 4, PT0S, null, 2023-07-16, 2023-07-16, null, 2023-07-16T23:01:54.059339538Z, 2023-07-16T23:01:54.059359638) (ArrowEncoderSuite.scala:194) [info] Analysis: [info] NullableData(instant: 2023-07-16T23:01:54.059339Z -> 2023-07-16T23:01:54.059339538Z, localDateTime: 2023-07-16T23:01:54.059359 -> 2023-07-16T23:01:54.059359638) [info] org.scalatest.exceptions.TestFailedException: ... [info] - lenient field serialization - timestamp/instant *** FAILED *** (26 milliseconds) [info] 2023-07-16T23:01:55.112838Z did not equal 2023-07-16T23:01:55.112838568Z (ArrowEncoderSuite.scala:194) [info] org.scalatest.exceptions.TestFailedException: ... ``` ### Does this PR introduce _any_ user-facing change? No, just for test ### How was this patch tested? - Pass GitHub Action - Git Hub Action test with Java 17 passed: https://github.com/LuciferYang/spark/actions/runs/5647253889/job/15297009685 <img width="1278" alt="image" src="https://github.com/apache/spark/assets/1475305/27a4350a-9475-45e3-b39f-b0b1e8f14e92"> Closes #42039 from LuciferYang/ArrowEncoderSuite-Java17. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Sean Owen <sro...@gmail.com> (cherry picked from commit da359259b138864a52ea98a4e19c55e593a5a8fa) Signed-off-by: Sean Owen <sro...@gmail.com> --- .../spark/sql/connect/client/arrow/ArrowEncoderSuite.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/arrow/ArrowEncoderSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/arrow/ArrowEncoderSuite.scala index 3f8ac1cb8d1..5c035a613fe 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/arrow/ArrowEncoderSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/arrow/ArrowEncoderSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.connect.client.arrow import java.math.BigInteger import java.time.{Duration, Period, ZoneOffset} +import java.time.temporal.ChronoUnit import java.util import java.util.{Collections, Objects} @@ -361,8 +362,10 @@ class ArrowEncoderSuite extends ConnectFunSuite with BeforeAndAfterAll { test("nullable fields") { val encoder = ScalaReflection.encoderFor[NullableData] - val instant = java.time.Instant.now() - val now = java.time.LocalDateTime.now() + // SPARK-44457: Similar to SPARK-42770, calling `truncatedTo(ChronoUnit.MICROS)` + // on `Instant.now()` and `LocalDateTime.now()` to ensure microsecond accuracy is used. + val instant = java.time.Instant.now().truncatedTo(ChronoUnit.MICROS) + val now = java.time.LocalDateTime.now().truncatedTo(ChronoUnit.MICROS) val today = java.time.LocalDate.now() roundTripAndCheckIdentical(encoder) { () => val maybeNull = MaybeNull(3) @@ -602,7 +605,9 @@ class ArrowEncoderSuite extends ConnectFunSuite with BeforeAndAfterAll { } test("lenient field serialization - timestamp/instant") { - val base = java.time.Instant.now() + // SPARK-44457: Similar to SPARK-42770, calling `truncatedTo(ChronoUnit.MICROS)` + // on `Instant.now()` to ensure microsecond accuracy is used. + val base = java.time.Instant.now().truncatedTo(ChronoUnit.MICROS) val instants = () => Iterator.tabulate(10)(i => base.plusSeconds(i * i * 60)) val timestamps = () => instants().map(java.sql.Timestamp.from) val combo = () => instants() ++ timestamps() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org