This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 16edb4ac9629 YARN-11630. Passing admin Java options to container
localizers (#6349)
16edb4ac9629 is described below
commit 16edb4ac9629206fcd52a05326f5e625715287e4
Author: Peter Szucs <116345192+p-sz...@users.noreply.github.com>
AuthorDate: Thu Dec 14 16:52:57 2023 +0100
YARN-11630. Passing admin Java options to container localizers (#6349)
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 7 +++
.../src/main/resources/yarn-default.xml | 9 ++++
.../localizer/ContainerLocalizer.java | 17 ++++--
.../localizer/TestContainerLocalizer.java | 62 ++++++++++++++++++++++
4 files changed, 92 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d195c8cdae68..0ab4107c1320 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2232,6 +2232,13 @@ public class YarnConfiguration extends Configuration {
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
+ /** The admin JVM options used on forking ContainerLocalizer process
+ by container executor. */
+ public static final String NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY =
+ NM_PREFIX + "container-localizer.admin.java.opts";
+
+ public static final String NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_DEFAULT =
"";
+
/*
* Flag to indicate whether JDK17's required add-exports flags should be
added to
* container localizers regardless of the user specified JAVA_OPTS.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b8892e4fc12e..41e38f601cbd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1488,6 +1488,15 @@
<value>-Xmx256m</value>
</property>
+ <property>
+ <description>
+ The admin JVM options used on forking ContainerLocalizer process
+ by the container executor.
+ </description>
+ <name>yarn.nodemanager.container-localizer.admin.java.opts</name>
+ <value></value>
+ </property>
+
<property>
<name>yarn.nodemanager.container-localizer.java.opts.add-exports-as-default</name>
<value>true</value>
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index c928b19874ed..0477e7320fee 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -47,6 +47,8 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
@@ -406,16 +408,25 @@ public class ContainerLocalizer {
* @param conf the configuration properties to launch the resource localizer.
*/
public static List<String> getJavaOpts(Configuration conf) {
- String opts =
conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
+ String adminOpts =
conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
+ YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_DEFAULT);
+ String userOpts =
conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
+
boolean isExtraJDK17OptionsConfigured =
conf.getBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT);
if (Shell.isJavaVersionAtLeast(17) && isExtraJDK17OptionsConfigured) {
- opts = opts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS);
+ userOpts = userOpts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS);
}
- return Arrays.asList(opts.split(" "));
+
+ List<String> adminOptionList = Arrays.asList(adminOpts.split("\\s+"));
+ List<String> userOptionList = Arrays.asList(userOpts.split("\\s+"));
+
+ return Stream.concat(adminOptionList.stream(), userOptionList.stream())
+ .filter(s -> !s.isEmpty())
+ .collect(Collectors.toList());
}
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
index 76aa73142e0c..8bde60e8db98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
@@ -737,4 +737,66 @@ static DataInputBuffer createFakeCredentials(Random r, int
nTok)
}
Assert.assertTrue(javaOpts.contains("-Xmx256m"));
}
+
+ @Test
+ public void testAdminOptionsPrecedeUserDefinedJavaOptions() throws Exception
{
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
+ "adminOption1 adminOption2");
+ conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
+ " userOption1 userOption2");
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ Assert.assertEquals(4, javaOpts.size());
+ Assert.assertTrue(javaOpts.get(0).equals("adminOption1"));
+ Assert.assertTrue(javaOpts.get(1).equals("adminOption2"));
+ Assert.assertTrue(javaOpts.get(2).equals("userOption1"));
+ Assert.assertTrue(javaOpts.get(3).equals("userOption2"));
+ }
+
+ @Test
+ public void testAdminOptionsPrecedeDefaultUserOptions() throws Exception {
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
+ "adminOption1 adminOption2");
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ Assert.assertEquals(3, javaOpts.size());
+ Assert.assertTrue(javaOpts.get(0).equals("adminOption1"));
+ Assert.assertTrue(javaOpts.get(1).equals("adminOption2"));
+ Assert.assertTrue(javaOpts.get(2).equals("-Xmx256m"));
+ }
+
+ @Test
+ public void testUserOptionsWhenAdminOptionsAreNotDefined() throws Exception {
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+ conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
+ "userOption1 userOption2");
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ Assert.assertEquals(2, javaOpts.size());
+ Assert.assertTrue(javaOpts.get(0).equals("userOption1"));
+ Assert.assertTrue(javaOpts.get(1).equals("userOption2"));
+ }
+
+ @Test
+ public void testJavaOptionsWithoutDefinedAdminOrUserOptions() throws
Exception {
+ ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
+ ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
+
+ Configuration conf = new Configuration();
+ List<String> javaOpts = localizer.getJavaOpts(conf);
+
+ Assert.assertEquals(1, javaOpts.size());
+ Assert.assertTrue(javaOpts.get(0).equals("-Xmx256m"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
