Repository: hadoop Updated Branches: refs/heads/branch-3.0 4010262ae -> 84297f69d
Revert "YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan" This reverts commit 874bdbc4013b72e40b64fb874e83d3d14045a93d. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84297f69 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84297f69 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84297f69 Branch: refs/heads/branch-3.0 Commit: 84297f69d8b5b8c125f4a0e82eb49b9121a96079 Parents: 4010262 Author: Jason Lowe <jl...@apache.org> Authored: Thu Feb 1 12:17:23 2018 -0600 Committer: Jason Lowe <jl...@apache.org> Committed: Thu Feb 1 12:17:23 2018 -0600 ---------------------------------------------------------------------- .../server/nodemanager/ContainerExecutor.java | 31 ++++++++++++-------- .../nodemanager/LinuxContainerExecutor.java | 8 +++++ .../launcher/ContainerLaunch.java | 15 ++-------- .../runtime/DefaultLinuxContainerRuntime.java | 6 ++++ .../DelegatingLinuxContainerRuntime.java | 11 +++++++ .../runtime/DockerLinuxContainerRuntime.java | 7 +++++ .../runtime/ContainerRuntime.java | 11 +++++++ .../launcher/TestContainerLaunch.java | 10 ++----- 8 files changed, 66 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index f5d454e..0b4dbc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -334,6 +334,7 @@ public abstract class ContainerExecutor implements Configurable { public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command, Path logDir, String user, String outFilename) throws IOException { + updateEnvForWhitelistVars(environment); ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create(); @@ -351,19 +352,6 @@ public abstract class ContainerExecutor implements Configurable { for (Map.Entry<String, String> env : environment.entrySet()) { sb.env(env.getKey(), env.getValue()); } - // Whitelist environment variables are treated specially. - // Only add them if they are not already defined in the environment. - // Add them using special syntax to prevent them from eclipsing - // variables that may be set explicitly in the container image (e.g, - // in a docker image) - for(String var : whitelistVars) { - if (!environment.containsKey(var)) { - String val = getNMEnvVar(var); - if (val != null) { - sb.whitelistedEnv(var, val); - } - } - } } if (resources != null) { @@ -663,6 +651,23 @@ public abstract class ContainerExecutor implements Configurable { } } + /** + * Propagate variables from the nodemanager's environment into the + * container's environment if unspecified by the container. + * @param env the environment to update + * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST + */ + protected void updateEnvForWhitelistVars(Map<String, String> env) { + for(String var : whitelistVars) { + if (!env.containsKey(var)) { + String val = getNMEnvVar(var); + if (val != null) { + env.put(var, val); + } + } + } + } + @VisibleForTesting protected String getNMEnvVar(String varname) { return System.getenv(varname); http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 349dd97..97f1a11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -63,6 +63,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -469,6 +470,13 @@ public class LinuxContainerExecutor extends ContainerExecutor { } @Override + protected void updateEnvForWhitelistVars(Map<String, String> env) { + if (linuxContainerRuntime.useWhitelistEnv(env)) { + super.updateEnvForWhitelistVars(env); + } + } + + @Override public int launchContainer(ContainerStartContext ctx) throws IOException, ConfigurationException { Container container = ctx.getContainer(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index e9d136e..c8ca2c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1077,9 +1077,6 @@ public class ContainerLaunch implements Callable<Integer> { public abstract void env(String key, String value) throws IOException; - public abstract void whitelistedEnv(String key, String value) - throws IOException; - public abstract void echo(String echoStr) throws IOException; public final void symlink(Path src, Path dst) throws IOException { @@ -1200,11 +1197,6 @@ public class ContainerLaunch implements Callable<Integer> { } @Override - public void whitelistedEnv(String key, String value) throws IOException { - line("export ", key, "=${", key, ":-", "\"", value, "\"}"); - } - - @Override public void echo(final String echoStr) throws IOException { line("echo \"" + echoStr + "\""); } @@ -1295,11 +1287,6 @@ public class ContainerLaunch implements Callable<Integer> { } @Override - public void whitelistedEnv(String key, String value) throws IOException { - env(key, value); - } - - @Override public void echo(final String echoStr) throws IOException { lineWithLenCheck("@echo \"", echoStr, "\""); } @@ -1398,6 +1385,8 @@ public class ContainerLaunch implements Callable<Integer> { environment.put(Environment.PWD.name(), pwd.toString()); + putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); + if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 6c3ae85..e9c58b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -72,6 +73,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { } @Override + public boolean useWhitelistEnv(Map<String, String> env) { + return true; + } + + @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { //nothing to do here at the moment. http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index 9fe4927..517a4e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -93,6 +93,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime { } } + @Override + public boolean useWhitelistEnv(Map<String, String> env) { + try { + LinuxContainerRuntime runtime = pickContainerRuntime(env); + return runtime.useWhitelistEnv(env); + } catch (ContainerExecutionException e) { + LOG.debug("Unable to determine runtime"); + return false; + } + } + @VisibleForTesting LinuxContainerRuntime pickContainerRuntime( Map<String, String> environment) throws ContainerExecutionException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 30a1e05..5e3e15c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -308,6 +308,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } @Override + public boolean useWhitelistEnv(Map<String, String> env) { + // Avoid propagating nodemanager environment variables into the container + // so those variables can be picked up from the Docker image instead. + return false; + } + + @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java index 7caa0ed..aa294fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java @@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import java.util.Map; + /** * An abstraction for various container runtime implementations. Examples * include Process Tree, Docker, Appc runtimes etc. These implementations @@ -83,4 +85,13 @@ public interface ContainerRuntime { * and hostname */ String[] getIpAndHost(Container container) throws ContainerExecutionException; + + /** + * Whether to propagate the whitelist of environment variables from the + * nodemanager environment into the container environment. + * @param env the container's environment variables + * @return true if whitelist variables should be propagated, false otherwise + * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST + */ + boolean useWhitelistEnv(Map<String, String> env); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/84297f69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 5e57bb5..5923f8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -337,8 +337,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); // Available in env and in whitelist Assert.assertTrue(shellContent.contains( - "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" - )); + "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\"")); fos.flush(); fos.close(); } @@ -383,12 +382,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest { // Whitelisted variable overridden by container Assert.assertTrue(shellContent.contains( "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\"")); - // Available in env but not in whitelist + // Verify no whitelisted variables inherited from NM env Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); - // Available in env and in whitelist - Assert.assertTrue(shellContent.contains( - "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" - )); + Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME")); fos.flush(); fos.close(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
