This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 85b3ea6f502 MAPREDUCE-7449: Add add-opens flag to container launch
commands on JDK17 nodes (#5935)
85b3ea6f502 is described below
commit 85b3ea6f50235e70fea6a39152e57fbb2010179e
Author: Benjamin Teke <brumi1...@users.noreply.github.com>
AuthorDate: Thu Aug 10 22:47:17 2023 +0200
MAPREDUCE-7449: Add add-opens flag to container launch commands on JDK17
nodes (#5935)
---
.../mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java | 5 +++--
.../src/main/java/org/apache/hadoop/mapred/JobConf.java | 8 ++++++++
.../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 9 +++++++++
.../src/main/resources/mapred-default.xml | 12 ++++++++++++
.../src/main/java/org/apache/hadoop/mapred/YARNRunner.java | 7 +++++++
.../org/apache/hadoop/yarn/api/ApplicationConstants.java | 7 +++++++
.../hadoop/yarn/applications/distributedshell/Client.java | 2 ++
.../containermanager/launcher/ContainerLaunch.java | 7 +++++++
.../containermanager/launcher/TestContainerLaunch.java | 9 ++++++---
9 files changed, 61 insertions(+), 5 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index f00ff281f30..e22bd44ee90 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -52,6 +52,7 @@ public class TestMapReduceChildJVM {
MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
Configuration conf = new Configuration();
conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT, false);
Job job = app.submit(conf);
app.waitForState(job, JobState.SUCCEEDED);
app.verifyCompleted();
@@ -123,7 +124,7 @@ public class TestMapReduceChildJVM {
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN " +
- " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") +
"/tmp" +
+ " -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" +
MRApps.crossPlatformify("PWD") + "/tmp" +
" -Dlog4j.configuration=container-log4j.properties" +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
@@ -165,7 +166,7 @@ public class TestMapReduceChildJVM {
"[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
" -Djava.net.preferIPv4Stack=true" +
" -Dhadoop.metrics.log.level=WARN " +
- " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp"
+
+ " -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" +
MRApps.crossPlatformify("PWD") + "/tmp" +
" -Dlog4j.configuration=" + testLogPropertieFile +
" -Dyarn.app.container.log.dir=<LOG_DIR>" +
" -Dyarn.app.container.log.filesize=0" +
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 895c9f022ef..db398e8dbdc 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2207,6 +2208,13 @@ public class JobConf extends Configuration {
javaOpts += " " + xmxArg;
}
+ // JDK17 support: automatically add
--add-opens=java.base/java.lang=ALL-UNNAMED
+ // so the tasks can launch on a JDK17 node.
+ if (getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT,
+ MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) {
+ javaOpts += " " + ApplicationConstants.JVM_ADD_OPENS_VAR;
+ }
+
return javaOpts;
}
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 02fa4eca636..8ec984e777b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -305,6 +305,15 @@ public interface MRJobConfig {
"os.name,os.version,java.home,java.runtime.version,java.vendor," +
"java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name";
+ /*
+ * Flag to indicate whether JDK17's required add-opens flags should be added
to MR AM and
+ * map/reduce containers regardless of the user specified java opts.
+ */
+ public static final String MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT =
+ "mapreduce.jvm.add-opens-as-default";
+
+ public static final boolean MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT = true;
+
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
public static final int DEFAULT_IO_SORT_FACTOR = 10;
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 2921b31caf9..a6d68acda34 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1804,6 +1804,18 @@
<description>Comma-delimited list of system properties to log on mapreduce
JVM start</description>
</property>
+<property>
+ <name>mapreduce.jvm.add-opens-as-default</name>
+ <value>true</value>
+ <description>Since on JDK17 it's no longer possible to use the reflection
API to
+ access non-public fields and methods add-opens flags should be added to MR
AM
+ and map/reduce containers regardless of the user specified java opts.
Setting
+ this to true will add the flags to the container launch commands on nodes
with
+ JDK17 or higher. Defaults to true, but the setting has no effect on nodes
using
+ JDK16 and before.
+ </description>
+</property>
+
<!-- jobhistory properties -->
<property>
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 084d373f1e2..12e8c7896e3 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -485,6 +485,13 @@ public class YARNRunner implements ClientProtocol {
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
+ // JDK17 support: automatically add
--add-opens=java.base/java.lang=ALL-UNNAMED
+ // so the tasks can launch on a JDK17 node.
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT,
+ MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) {
+ vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR);
+ }
+
if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE,
MRJobConfig.DEFAULT_MR_AM_PROFILE)) {
final String profileParams =
jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 533eaddabb4..31ba5a9200c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -77,6 +77,13 @@ public interface ApplicationConstants {
String APPLICATION_WEB_PROXY_BASE_ENV =
"APPLICATION_WEB_PROXY_BASE";
+ /**
+ * The environmental variable for JDK17's add-opens workaround. This
+ * should be replaced either a correctly formatted add-opens option if JDK17
is used
+ * or an empty string if not on container launch.
+ */
+ String JVM_ADD_OPENS_VAR = "<ADD_OPENS>";
+
/**
* The temporary environmental variable for container log directory. This
* should be replaced by real container log directory on container launch.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 098f3981cfd..2987165486e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -952,6 +952,8 @@ public class Client {
vargs.add("\"" + Environment.JAVA_HOME.$$() + "/bin/java\"");
// Set Xmx based on am memory size
vargs.add("-Xmx" + amMemory + "m");
+ // JDK17 support
+ vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR);
// Set class name
vargs.add(appMasterMainClass);
// Set params for Application Master
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 7d91e5d395f..143086db2ae 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -170,6 +170,13 @@ public class ContainerLaunch implements Callable<Integer> {
var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
File.pathSeparator);
+ if (Shell.isJavaVersionAtLeast(17)) {
+ var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR,
+ "--add-opens=java.base/java.lang=ALL-UNNAMED");
+ } else {
+ var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, "");
+ }
+
// replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
// as %VAR% and on Linux replaced as "$VAR"
if (Shell.WINDOWS) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 15dac3daa54..bd135ff5193 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -575,18 +575,21 @@ public class TestContainerLaunch extends
BaseContainerManagerTest {
+ Apps.crossPlatformify("HADOOP_HOME") +
"/share/hadoop/common/lib/*"
+ ApplicationConstants.CLASS_PATH_SEPARATOR
+ Apps.crossPlatformify("HADOOP_LOG_HOME")
- + ApplicationConstants.LOG_DIR_EXPANSION_VAR;
+ + ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ + " " + ApplicationConstants.JVM_ADD_OPENS_VAR;
String res = ContainerLaunch.expandEnvironment(input, logPath);
+ String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ?
+ "--add-opens=java.base/java.lang=ALL-UNNAMED" : "";
if (Shell.WINDOWS) {
Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
+ "%HADOOP_HOME%/share/hadoop/common/lib/*;"
- + "%HADOOP_LOG_HOME%/nm/container/logs", res);
+ + "%HADOOP_LOG_HOME%/nm/container/logs" + " " + expectedAddOpens,
res);
} else {
Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
+ "$HADOOP_HOME/share/hadoop/common/lib/*:"
- + "$HADOOP_LOG_HOME/nm/container/logs", res);
+ + "$HADOOP_LOG_HOME/nm/container/logs" + " " + expectedAddOpens,
res);
}
System.out.println(res);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
