YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw)
Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e06811e6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e06811e6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e06811e6 Branch: refs/heads/YARN-2928 Commit: e06811e689447cb30418ae66f1e39fe9ff2ecbb3 Parents: 0cd01a1 Author: Allen Wittenauer <a...@apache.org> Authored: Wed May 27 16:40:56 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue Jun 2 16:12:32 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/api/ApplicationConstants.java | 50 +++++++++----------- .../resourcemanager/amlauncher/AMLauncher.java | 29 +++++------- .../TestApplicationMasterLauncher.java | 7 ++- 4 files changed, 41 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c8dbb08..54b33bc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -136,6 +136,9 @@ Trunk - Unreleased YARN-2428. LCE default banned user list should have yarn (Varun Saxena via aw) + YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var + for a container (Darrell Taylor via aw) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 42464da..d75297a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -28,13 +28,13 @@ import org.apache.hadoop.util.Shell; /** * This is the API for the applications comprising of constants that YARN sets * up for the applications and the containers. - * + * * TODO: Investigate the semantics and security of each cross-boundary refs. */ @Public @Evolving public interface ApplicationConstants { - + /** * The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment * only @@ -48,11 +48,11 @@ public interface ApplicationConstants { UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; /** - * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in + * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in * ApplicationMaster's environment only. This states that for all non-relative * web URLs in the app masters web UI what base should they have. */ - public static final String APPLICATION_WEB_PROXY_BASE_ENV = + public static final String APPLICATION_WEB_PROXY_BASE_ENV = "APPLICATION_WEB_PROXY_BASE"; /** @@ -99,15 +99,9 @@ public interface ApplicationConstants { public static final String STDOUT = "stdout"; /** - * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment - * only - */ - public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS"; - - /** * Environment for Applications. - * - * Some of the environment variables for applications are <em>final</em> + * + * Some of the environment variables for applications are <em>final</em> * i.e. they cannot be modified by the applications. */ public enum Environment { @@ -116,76 +110,76 @@ public interface ApplicationConstants { * Final, non-modifiable. */ USER("USER"), - + /** * $LOGNAME * Final, non-modifiable. */ LOGNAME("LOGNAME"), - + /** * $HOME * Final, non-modifiable. */ HOME("HOME"), - + /** * $PWD * Final, non-modifiable. */ PWD("PWD"), - + /** * $PATH */ PATH("PATH"), - + /** * $SHELL */ SHELL("SHELL"), - + /** * $JAVA_HOME */ JAVA_HOME("JAVA_HOME"), - + /** * $CLASSPATH */ CLASSPATH("CLASSPATH"), - + /** * $APP_CLASSPATH */ APP_CLASSPATH("APP_CLASSPATH"), - + /** * $LD_LIBRARY_PATH */ LD_LIBRARY_PATH("LD_LIBRARY_PATH"), - + /** * $HADOOP_CONF_DIR * Final, non-modifiable. */ HADOOP_CONF_DIR("HADOOP_CONF_DIR"), - + /** * $HADOOP_COMMON_HOME */ HADOOP_COMMON_HOME("HADOOP_COMMON_HOME"), - + /** * $HADOOP_HDFS_HOME */ HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"), - + /** * $MALLOC_ARENA_MAX */ MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"), - + /** * $HADOOP_YARN_HOME */ @@ -240,11 +234,11 @@ public interface ApplicationConstants { private Environment(String variable) { this.variable = variable; } - + public String key() { return variable; } - + public String toString() { return variable; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index f62ee50..42bf579 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -80,10 +80,10 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; - + @SuppressWarnings("rawtypes") private final EventHandler handler; - + public AMLauncher(RMContext rmContext, RMAppAttempt application, AMLauncherEventType eventType, Configuration conf) { this.application = application; @@ -93,20 +93,20 @@ public class AMLauncher implements Runnable { this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); } - + private void connect() throws IOException { ContainerId masterContainerID = masterContainer.getId(); - + containerMgrProxy = getContainerMgrProxy(masterContainerID); } - + private void launch() throws IOException, YarnException { connect(); ContainerId masterContainerID = masterContainer.getId(); ApplicationSubmissionContext applicationContext = application.getSubmissionContext(); LOG.info("Setting up container " + masterContainer - + " for AM " + application.getAppAttemptId()); + + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); @@ -130,7 +130,7 @@ public class AMLauncher implements Runnable { + application.getAppAttemptId()); } } - + private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); @@ -188,17 +188,17 @@ public class AMLauncher implements Runnable { ContainerId containerID) throws IOException { // Construct the actual Container - ContainerLaunchContext container = + ContainerLaunchContext container = applicationMasterContext.getAMContainerSpec(); LOG.info("Command to launch container " + containerID + " : " + StringUtils.arrayToString(container.getCommands().toArray( new String[0]))); - + // Finalize the container setupTokens(container, containerID); - + return container; } @@ -210,7 +210,7 @@ public class AMLauncher implements Runnable { Map<String, String> environment = container.getEnvironment(); environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase()); - // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. + // Set AppSubmitTime to be consumable by the AM. ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); environment.put( @@ -218,9 +218,6 @@ public class AMLauncher implements Runnable { String.valueOf(rmContext.getRMApps() .get(applicationId) .getSubmitTime())); - environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV, - String.valueOf(rmContext.getRMApps().get( - applicationId).getMaxAppAttempts())); // Set flow context info for (String tag : rmContext.getRMApps().get(applicationId).getApplicationTags()) { @@ -267,7 +264,7 @@ public class AMLauncher implements Runnable { ((RMAppAttemptImpl)application).setAMRMToken(amrmToken); return amrmToken; } - + @SuppressWarnings("unchecked") public void run() { switch (eventType) { @@ -297,7 +294,7 @@ public class AMLauncher implements Runnable { sb.append(" is not handled by this NodeManager"); if (!e.getMessage().contains(sb.toString())) { // Ignoring if container is already killed by Node Manager. - LOG.info("Error cleaning master ", e); + LOG.info("Error cleaning master ", e); } } break; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 9a4395e..d4f8e93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -107,8 +107,7 @@ public class TestApplicationMasterLauncher { nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); - maxAppAttempts = - Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); + maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS; return StartContainersResponse.newInstance( new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(), new HashMap<ContainerId, SerializedException>()); @@ -184,8 +183,8 @@ public class TestApplicationMasterLauncher { am.waitForState(RMAppAttemptState.FINISHED); rm.stop(); } - - + + @SuppressWarnings("unused") @Test(timeout = 100000) public void testallocateBeforeAMRegistration() throws Exception {