Revert 'YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe.'
This reverts commit 729d05f5293acf63e7e4aa3bfbf29b999c9a2906. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3c44195 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3c44195 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3c44195 Branch: refs/heads/YARN-1011 Commit: a3c44195bed724c02bb76859fe2690d6a9e8f2e9 Parents: 729d05f Author: Eric Payne <epa...@apache.org> Authored: Wed Sep 13 14:38:58 2017 -0500 Committer: Eric Payne <epa...@apache.org> Committed: Wed Sep 13 14:38:58 2017 -0500 ---------------------------------------------------------------------- .../launcher/ContainerLaunch.java | 5 ++- .../launcher/TestContainerLaunch.java | 32 ++++++-------------- 2 files changed, 13 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c44195/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d0ce787..50443f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1148,7 +1148,10 @@ public class ContainerLaunch implements Callable<Integer> { environment.put(Environment.PWD.name(), pwd.toString()); - putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); + putEnvIfNotNull(environment, + Environment.HADOOP_CONF_DIR.name(), + System.getenv(Environment.HADOOP_CONF_DIR.name()) + ); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c44195/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 7176942..085b60f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -46,7 +46,6 @@ import java.util.StringTokenizer; import java.util.jar.JarFile; import java.util.jar.Manifest; -import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; @@ -55,7 +54,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.SecretManager.InvalidToken; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; @@ -726,15 +724,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); - final String userConfDir = "user_set_HADOOP_CONF_DIR"; - userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir); containerLaunchContext.setEnvironment(userSetEnv); File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); File processStartFile = - new File(tmpDir, "env_vars.tmp").getAbsoluteFile(); - final File processFinalFile = new File(tmpDir, "env_vars.txt").getAbsoluteFile(); if (Shell.WINDOWS) { fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> " @@ -755,8 +749,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.println("@echo " + Environment.HOME.$() + ">> " + processStartFile); - fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> " - + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -764,8 +756,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); } fileWriter.println("@echo " + cId + ">> " + processStartFile); - fileWriter.println("@move /Y " + processStartFile + " " - + processFinalFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.write("\numask 0"); // So that start file is readable by the test @@ -787,8 +777,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.write("\necho $" + Environment.HOME.name() + " >> " + processStartFile); - fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> " - + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -796,7 +784,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); } fileWriter.write("\necho $$ >> " + processStartFile); - fileWriter.write("\nmv " + processStartFile + " " + processFinalFile); fileWriter.write("\nexec sleep 100"); } fileWriter.close(); @@ -830,12 +817,13 @@ public class TestContainerLaunch extends BaseContainerManagerTest { StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - return processFinalFile.exists(); - } - }, 10, 20000); + int timeoutSecs = 0; + while (!processStartFile.exists() && timeoutSecs++ < 20) { + Thread.sleep(1000); + LOG.info("Waiting for process start-file to be created"); + } + Assert.assertTrue("ProcessStartFile doesn't exist!", + processStartFile.exists()); // Now verify the contents of the file List<String> localDirs = dirsHandler.getLocalDirs(); @@ -855,7 +843,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir); } BufferedReader reader = - new BufferedReader(new FileReader(processFinalFile)); + new BufferedReader(new FileReader(processStartFile)); Assert.assertEquals(cId.toString(), reader.readLine()); Assert.assertEquals(context.getNodeId().getHost(), reader.readLine()); Assert.assertEquals(String.valueOf(context.getNodeId().getPort()), @@ -878,7 +866,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { YarnConfiguration.NM_USER_HOME_DIR, YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), reader.readLine()); - Assert.assertEquals(userConfDir, reader.readLine()); + for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) { Assert.assertEquals( containerManager.getAuxServiceMetaData().get(serviceName), @@ -917,8 +905,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), containerLaunchContext.getEnvironment() .get(Environment.HOME.name())); - Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment() - .get(Environment.HADOOP_CONF_DIR.name())); // Get the pid of the process String pid = reader.readLine().trim(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org