This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 3a9185964a0 [SPARK-44092][CORE] Add `Utils.isJavaVersionAtLeast21` and make `core` module pass with Java 21 3a9185964a0 is described below commit 3a9185964a0de3c720a6b77d38a446258b73468e Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Mon Jun 19 09:57:14 2023 +0900 [SPARK-44092][CORE] Add `Utils.isJavaVersionAtLeast21` and make `core` module pass with Java 21 ### What changes were proposed in this pull request? This PR aims to make `core` module tests succeed in Java 21. To do that, this PR - Adds a utility variable `Utils.isJavaVersionAtLeast21` because Apache Commons Lang3 `3.12.0` doesn't have a constant for Java 21 yet. - Fix `UtilsSuite` according to the Java behavior change of `Files.createDirectories` API. ### Why are the changes needed? Java 20+ changes the behavior. - https://github.com/openjdk/jdk/commit/169a5d48afbc6627f36a768c17c2a5e56219d9c7 ``` 8294193: Files.createDirectories throws FileAlreadyExistsException for a symbolic link whose target is an existing directory ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual tests in Java 21. **JAVA** ``` $ java -version openjdk version "21-ea" 2023-09-19 OpenJDK Runtime Environment (build 21-ea+27-2343) OpenJDK 64-Bit Server VM (build 21-ea+27-2343, mixed mode, sharing) ``` **BEFORE** ``` $ $ build/sbt "core/test" -Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest ... [info] *** 1 TEST FAILED *** [error] Failed: Total 3451, Failed 1, Errors 0, Passed 3450, Ignored 10, Canceled 5 [error] Failed tests: [error] org.apache.spark.util.UtilsSuite [error] (core / Test / test) sbt.TestsFailedException: Tests unsuccessful [error] Total time: 1040 s (17:20), completed Jun 18, 2023, 12:27:59 AM ``` **AFTER** ``` $ build/sbt "core/testOnly org.apache.spark.util.UtilsSuite" ... [info] All tests passed. [success] Total time: 29 s, completed Jun 17, 2023, 11:16:23 PM ``` Closes #41648 from dongjoon-hyun/SPARK-44092. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- core/src/main/scala/org/apache/spark/util/Utils.scala | 6 ++++++ core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index c785c135a45..6e8f2c496e8 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1972,6 +1972,12 @@ private[spark] object Utils extends Logging with SparkClassUtils { */ val isMac = SystemUtils.IS_OS_MAC_OSX + /** + * Whether the underlying Java version is at least 21. + */ + val isJavaVersionAtLeast21 = + System.getProperty("java.version").split("[+.\\-]+", 3)(0).toInt >= 21 + /** * Whether the underlying operating system is Mac OS X and processor is Apple Silicon. */ diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index a2990f087b7..7923e81949d 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -527,7 +527,11 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties { // 6. Symbolic link val scenario6 = java.nio.file.Files.createSymbolicLink(new File(testDir, "scenario6") .toPath, scenario1.toPath).toFile - assert(!Utils.createDirectory(scenario6)) + if (Utils.isJavaVersionAtLeast21) { + assert(Utils.createDirectory(scenario6)) + } else { + assert(!Utils.createDirectory(scenario6)) + } assert(scenario6.exists()) // 7. Directory exists --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org