This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new 146fba1a22e [SPARK-45508][CORE] Add "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" so Platform can access Cleaner on Java 9+ 146fba1a22e is described below commit 146fba1a22e3f1555f3e4494522810030f9a7854 Author: Josh Rosen <joshro...@databricks.com> AuthorDate: Fri Oct 13 13:29:45 2023 +0800 [SPARK-45508][CORE] Add "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" so Platform can access Cleaner on Java 9+ This PR adds `--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED` to our JVM flags so that we can access `jdk.internal.ref.Cleaner` in JDK 9+. This allows Spark to allocate direct memory while ignoring the JVM's MaxDirectMemorySize limit. Spark uses JDK internal APIs to directly construct DirectByteBuffers while bypassing that limit, but there is a fallback path at https://github.com/apache/spark/blob/v3.5.0/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java#L213 that is used if we cannot reflectively access the `Cleaner` API. No. Added a unit test in `PlatformUtilSuite`. No. Closes #43344 from JoshRosen/SPARK-45508. Authored-by: Josh Rosen <joshro...@databricks.com> Signed-off-by: yangjie01 <yangji...@baidu.com> (cherry picked from commit 96bac6c033b5bb37101ebcd8436ab9a84db8e092) Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java | 7 ++++++- .../src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java | 7 +++++++ .../src/main/java/org/apache/spark/launcher/JavaModuleOptions.java | 1 + pom.xml | 1 + project/SparkBuild.scala | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index a91ea2ee6b5..e02346c4773 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -96,7 +96,7 @@ public final class Platform { Method createMethod = cleanerClass.getMethod("create", Object.class, Runnable.class); // Accessing jdk.internal.ref.Cleaner should actually fail by default in JDK 9+, // unfortunately, unless the user has allowed access with something like - // --add-opens java.base/java.lang=ALL-UNNAMED If not, we can't really use the Cleaner + // --add-opens java.base/jdk.internal.ref=ALL-UNNAMED If not, we can't use the Cleaner // hack below. It doesn't break, just means the user might run into the default JVM limit // on off-heap memory and increase it or set the flag above. This tests whether it's // available: @@ -118,6 +118,11 @@ public final class Platform { } } + // Visible for testing + public static boolean cleanerCreateMethodIsDefined() { + return CLEANER_CREATE_METHOD != null; + } + /** * @return true when running JVM is having sun's Unsafe package available in it and underlying * system having unaligned-access capability. diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java index c59878fea99..c99f2d85f4e 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java @@ -157,4 +157,11 @@ public class PlatformUtilSuite { Assert.assertEquals(1024 * 1024 + 7, onheap4.size()); Assert.assertEquals(obj3, onheap4.getBaseObject()); } + + @Test + public void cleanerCreateMethodIsDefined() { + // Regression test for SPARK-45508: we don't expect the "no cleaner" fallback + // path to be hit in normal usage. + Assert.assertTrue(Platform.cleanerCreateMethodIsDefined()); + } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java index 013dde2766f..f6a9607e7c5 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java +++ b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java @@ -36,6 +36,7 @@ public class JavaModuleOptions { "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", "--add-opens=java.base/sun.security.action=ALL-UNNAMED", diff --git a/pom.xml b/pom.xml index 8fc4b89a78c..be8400c33bf 100644 --- a/pom.xml +++ b/pom.xml @@ -308,6 +308,7 @@ --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 563d5357754..718f2bb28ce 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1619,6 +1619,7 @@ object TestSettings { "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", "--add-opens=java.base/sun.security.action=ALL-UNNAMED", --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org