YARN-5877. Allow all env's from yarn.nodemanager.env-whitelist to get overridden during launch. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/575773a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/575773a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/575773a3 Branch: refs/heads/YARN-5734 Commit: 575773a3570b85293fdf7b8aeb8467580ec7f896 Parents: d2026ae Author: Sunil G <sun...@apache.org> Authored: Tue Dec 20 11:39:06 2016 +0530 Committer: Sunil G <sun...@apache.org> Committed: Tue Dec 20 11:39:06 2016 +0530 ---------------------------------------------------------------------- .../server/nodemanager/ContainerExecutor.java | 11 ++-- .../launcher/TestContainerLaunch.java | 63 ++++++++++++++++---- 2 files changed, 58 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/575773a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index f880506..158585e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -44,7 +44,6 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -320,11 +319,11 @@ public abstract class ContainerExecutor implements Configurable { ContainerLaunch.ShellScriptBuilder.create(); Set<String> whitelist = new HashSet<>(); - whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name()); - whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name()); - whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name()); - whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name()); - whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name()); + String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST, + YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(","); + for (String param : nmWhiteList) { + whitelist.add(param); + } if (environment != null) { for (Map.Entry<String, String> env : environment.entrySet()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/575773a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 4ce816a..23b99d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows; +import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -33,6 +34,9 @@ import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -172,10 +176,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { commands.add("/bin/sh ./\\\"" + badSymlink + "\\\""); } - new DefaultContainerExecutor() - .writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user", - tempFile.getName()); + DefaultContainerExecutor defaultContainerExecutor = + new DefaultContainerExecutor(); + defaultContainerExecutor.setConf(new YarnConfiguration()); + defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName()); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -242,9 +247,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { } else { commands.add("/bin/sh ./\\\"" + symLink + "\\\""); } - new DefaultContainerExecutor() - .writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + DefaultContainerExecutor defaultContainerExecutor = + new DefaultContainerExecutor(); + defaultContainerExecutor.setConf(new YarnConfiguration()); + defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user"); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -279,6 +286,39 @@ public class TestContainerLaunch extends BaseContainerManagerTest { } } + @Test(timeout = 20000) + public void testWriteEnvExport() throws Exception { + // Valid only for unix + assumeNotWindows(); + File shellFile = Shell.appendScriptExtension(tmpDir, "hello"); + Map<String, String> env = new HashMap<String, String>(); + env.put("HADOOP_COMMON_HOME", "/opt/hadoopcommon"); + env.put("HADOOP_MAPRED_HOME", "/opt/hadoopbuild"); + Map<Path, List<String>> resources = new HashMap<Path, List<String>>(); + FileOutputStream fos = new FileOutputStream(shellFile); + List<String> commands = new ArrayList<String>(); + DefaultContainerExecutor defaultContainerExecutor = + new DefaultContainerExecutor(); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_ENV_WHITELIST, + "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME"); + defaultContainerExecutor.setConf(conf); + defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user"); + String shellContent = + new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())), + StandardCharsets.UTF_8); + Assert.assertTrue(shellContent + .contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\"")); + // Not available in env and whitelist + Assert.assertTrue(shellContent.contains("export HADOOP_MAPRED_HOME=" + + "${HADOOP_MAPRED_HOME:-\"/opt/hadoopbuild\"}")); + // Not available in env but in whitelist + Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME")); + fos.flush(); + fos.close(); + } + @Test (timeout = 20000) public void testInvalidEnvSyntaxDiagnostics() throws IOException { @@ -297,9 +337,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { "\"workflowName\":\"\n\ninsert table " + "\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " ); List<String> commands = new ArrayList<String>(); - new DefaultContainerExecutor() - .writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + DefaultContainerExecutor defaultContainerExecutor = + new DefaultContainerExecutor(); + defaultContainerExecutor.setConf(new YarnConfiguration()); + defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user"); fos.flush(); fos.close(); @@ -377,6 +419,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { List<String> commands = new ArrayList<String>(); commands.add(command); ContainerExecutor exec = new DefaultContainerExecutor(); + exec.setConf(new YarnConfiguration()); exec.writeLaunchEnv(fos, env, resources, commands, new Path(localLogDir.getAbsolutePath()), "user"); fos.flush(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org