This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2d871fab7888 MAPREDUCE-7456. Extend add-opens flag to container launch
commands on JDK17 nodes. Contributed by Peter Szucs
2d871fab7888 is described below
commit 2d871fab7888aa531a199aa52bc1e0875f6a85f3
Author: Szilard Nemeth <snem...@apache.org>
AuthorDate: Wed Sep 27 22:33:21 2023 -0400
MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17
nodes. Contributed by Peter Szucs
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 +++++
.../src/main/resources/yarn-default.xml | 11 +++++++
.../containermanager/launcher/ContainerLaunch.java | 8 +++--
.../localizer/ContainerLocalizer.java | 15 ++++++++++
.../launcher/TestContainerLaunch.java | 8 +++--
.../localizer/TestContainerLocalizer.java | 35 ++++++++++++++++++++++
6 files changed, 81 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 13d4c209110e..90a8978a228b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2232,6 +2232,14 @@ public class YarnConfiguration extends Configuration {
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
+ /*
+ * Flag to indicate whether JDK17's required add-exports flags should be
added to
+ * container localizers regardless of the user specified JAVA_OPTS.
+ */
+ public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY =
+ NM_PREFIX + "container-localizer.java.opts.add-exports-as-default";
+ public static final boolean
NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT = true;
+
/** The log level of container localizer process. */
public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
NM_PREFIX + "container-localizer.log.level";
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2259b73fb651..9991e841d74b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1484,6 +1484,17 @@
<value>-Xmx256m</value>
</property>
+ <property>
+
<name>yarn.nodemanager.container-localizer.java.opts.add-exports-as-default</name>
+ <value>true</value>
+ <description>Since on JDK17 it is no longer possible to use the reflection
API to
+ access public fields and methods, add-exports flags should be added to
container
+ localizers regardless of the user specified JAVA_OPTS. Setting this to
true will
+ add the flags to the container localizer on nodes with JDK17 or higher.
+ Defaults to true, but the setting has no effect on nodes using JDK16 and
before.
+ </description>
+ </property>
+
<property>
<description>
The log level for container localizer while it is an independent process.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 143086db2aec..5466182b715d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -123,6 +123,11 @@ public class ContainerLaunch implements Callable<Integer> {
private static final String PID_FILE_NAME_FMT = "%s.pid";
static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
+ private static final String ADDITIONAL_JDK17_PLUS_OPTIONS =
+ "--add-opens=java.base/java.lang=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.util=ALL-UNNAMED";
+
protected final Dispatcher dispatcher;
protected final ContainerExecutor exec;
protected final Application app;
@@ -171,8 +176,7 @@ public class ContainerLaunch implements Callable<Integer> {
File.pathSeparator);
if (Shell.isJavaVersionAtLeast(17)) {
- var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR,
- "--add-opens=java.base/java.lang=ALL-UNNAMED");
+ var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR,
ADDITIONAL_JDK17_PLUS_OPTIONS);
} else {
var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, "");
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 604a810ec946..c928b19874ed 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -103,6 +103,14 @@ public class ContainerLocalizer {
new FsPermission((short) 0755);
public static final String CSI_VOLIUME_MOUNTS_ROOT = "csivolumes";
+ /*
+ * Testing discovered that these Java options are needed for Spark service
+ * running on JDK17 and Isilon clusters.
+ */
+ private static final String ADDITIONAL_JDK17_PLUS_OPTIONS =
+ "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.util=ALL-UNNAMED";
+
private final String user;
private final String appId;
private final List<Path> localDirs;
@@ -400,6 +408,13 @@ public class ContainerLocalizer {
public static List<String> getJavaOpts(Configuration conf) {
String opts =
conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
+ boolean isExtraJDK17OptionsConfigured =
+
conf.getBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY,
+
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT);
+
+ if (Shell.isJavaVersionAtLeast(17) && isExtraJDK17OptionsConfigured) {
+ opts = opts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS);
+ }
return Arrays.asList(opts.split(" "));
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 6971d34b9d81..6a9dd6b7476b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -580,8 +580,12 @@ public class TestContainerLaunch extends
BaseContainerManagerTest {
String res = ContainerLaunch.expandEnvironment(input, logPath);
- String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ?
- "--add-opens=java.base/java.lang=ALL-UNNAMED" : "";
+ String additionalJdk17PlusOptions =
+ "--add-opens=java.base/java.lang=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.util=ALL-UNNAMED";
+ String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ?
additionalJdk17PlusOptions : "";
+
if (Shell.WINDOWS) {
Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
+ "%HADOOP_HOME%/share/hadoop/common/lib/*;"
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
index 8d8f748347c9..76aa73142e0c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
@@ -702,4 +702,39 @@ static DataInputBuffer createFakeCredentials(Random r, int
nTok)
lfs.getFileStatus(destDirPath.getParent().getParent()).getPermission());
}
+ @Test
+ public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreConfigured()
throws Exception {
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+
conf.setBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY,
+ true);
+
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ if (Shell.isJavaVersionAtLeast(17)) {
+
Assert.assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED"));
+
Assert.assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED"));
+ }
+ Assert.assertTrue(javaOpts.contains("-Xmx256m"));
+ }
+
+ @Test
+ public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreNotConfigured()
throws Exception {
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+
conf.setBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY,
+ false);
+
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ if (Shell.isJavaVersionAtLeast(17)) {
+
Assert.assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED"));
+
Assert.assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED"));
+ }
+ Assert.assertTrue(javaOpts.contains("-Xmx256m"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
