Author: vinodkv Date: Wed Apr 24 00:01:08 2013 New Revision: 1471207 URL: http://svn.apache.org/r1471207 Log: YARN-561. Modified NodeManager to set key information into the environment of every container that it launches. Contributed by Xuan Gong. MAPREDUCE-5175. Updated MR App to not set envs that will be set by NMs anyways after YARN-561. Contributed by Xuan Gong.
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Apr 24 00:01:08 2013 @@ -47,6 +47,9 @@ Release 2.0.5-beta - UNRELEASED instead of extracting and populating information itself to start any container. (vinodkv) + MAPREDUCE-5175. Updated MR App to not set envs that will be set by NMs + anyways after YARN-561. (Xuan Gong via vinodkv) + OPTIMIZATIONS MAPREDUCE-4974. Optimising the LineRecordReader initialize() method Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java Wed Apr 24 00:01:08 2013 @@ -52,6 +52,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.service.AbstractService; /** @@ -280,7 +281,7 @@ public class LocalContainerLauncher exte // Use the AM's local dir env to generate the intermediate step // output files String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIRS.name())); conf.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for uber task: " + conf.get(MRConfig.LOCAL_DIR)); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Wed Apr 24 00:01:08 2013 @@ -111,8 +111,6 @@ public class MapReduceChildJVM { MRJobConfig.STDERR_LOGFILE_ENV, getTaskLogFile(TaskLog.LogName.STDERR) ); - environment.put(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV, - conf.get(MRJobConfig.APPLICATION_ATTEMPT_ID).toString()); } private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Wed Apr 24 00:01:08 2013 @@ -57,6 +57,9 @@ import org.apache.hadoop.util.DiskChecke import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.LogManager; /** @@ -216,7 +219,7 @@ class YarnChild { */ private static void configureLocalDirs(Task task, JobConf job) throws IOException { String[] localSysDirs = StringUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.LOCAL_DIR_ENV)); + System.getenv(Environment.LOCAL_DIRS.name())); job.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR)); LocalDirAllocator lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); @@ -256,12 +259,14 @@ class YarnChild { final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE); job.setCredentials(credentials); - String appAttemptIdEnv = System - .getenv(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV); - LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv); + ApplicationAttemptId appAttemptId = + ConverterUtils.toContainerId( + System.getenv(Environment.CONTAINER_ID.name())) + .getApplicationAttemptId(); + LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId); // Set it in conf, so as to be able to be used the the OutputCommitter. - job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, Integer - .parseInt(appAttemptIdEnv)); + job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, + appAttemptId.getAttemptId()); // set tcp nodelay job.setBoolean("ipc.client.tcpnodelay", true); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Wed Apr 24 00:01:08 2013 @@ -116,6 +116,7 @@ import org.apache.hadoop.yarn.SystemCloc import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -1270,22 +1271,22 @@ public class MRAppMaster extends Composi try { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = - System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); - String nodeHostString = System.getenv(ApplicationConstants.NM_HOST_ENV); - String nodePortString = System.getenv(ApplicationConstants.NM_PORT_ENV); + System.getenv(Environment.CONTAINER_ID.name()); + String nodeHostString = System.getenv(Environment.NM_HOST.name()); + String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = - System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); + System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); String maxAppAttempts = System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); validateInputParam(containerIdStr, - ApplicationConstants.AM_CONTAINER_ID_ENV); - validateInputParam(nodeHostString, ApplicationConstants.NM_HOST_ENV); - validateInputParam(nodePortString, ApplicationConstants.NM_PORT_ENV); + Environment.CONTAINER_ID.name()); + validateInputParam(nodeHostString, Environment.NM_HOST.name()); + validateInputParam(nodePortString, Environment.NM_PORT.name()); validateInputParam(nodeHttpPortString, - ApplicationConstants.NM_HTTP_PORT_ENV); + Environment.NM_HTTP_PORT.name()); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(maxAppAttempts, Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1471207&r1=1471206&r2=1471207&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Apr 24 00:01:08 2013 @@ -569,8 +569,6 @@ public interface MRJobConfig { public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; - public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID_ENV"; - // This should be the directory where splits file gets localized on the node // running ApplicationMaster. public static final String JOB_SUBMIT_DIR = "jobSubmitDir";