Author: cnauroth Date: Fri Oct 4 05:37:54 2013 New Revision: 1529077 URL: http://svn.apache.org/r1529077 Log: MAPREDUCE-5442. $HADOOP_MAPRED_HOME/$HADOOP_CONF_DIR setting not working on Windows. Contributed by Yingda Chen.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1529077&r1=1529076&r2=1529077&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Oct 4 05:37:54 2013 @@ -275,6 +275,9 @@ Release 2.1.2 - UNRELEASED MAPREDUCE-5489. MR jobs hangs as it does not use the node-blacklisting feature in RM requests (Zhijie Shen via bikas) + MAPREDUCE-5442. $HADOOP_MAPRED_HOME/$HADOOP_CONF_DIR setting not working on + Windows. (Yingda Chen via cnauroth) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1529077&r1=1529076&r2=1529077&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Fri Oct 4 05:37:54 2013 @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -181,7 +182,8 @@ public class MRApps extends Apps { boolean foundFrameworkInClasspath = (frameworkName == null); for (String c : conf.getStrings( MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, - MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { + StringUtils.getStrings( + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))){ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c .trim()); if (!foundFrameworkInClasspath) { Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1529077&r1=1529076&r2=1529077&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Fri Oct 4 05:37:54 2013 @@ -191,7 +191,9 @@ public class TestMRApps { } assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath)); String mrAppClasspath = - job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH); + job.getConfiguration().get( + MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH); if (mrAppClasspath != null) { mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator) .trim(); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1529077&r1=1529076&r2=1529077&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Fri Oct 4 05:37:54 2013 @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.util.Shell; @InterfaceAudience.Private @InterfaceStability.Evolving @@ -658,10 +659,12 @@ public interface MRJobConfig { /** * Default CLASSPATH for all YARN MapReduce applications. */ - public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = { - "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*", - "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*", - }; + public final String + DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = Shell.WINDOWS ? + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*," + + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*" : + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*," + + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*"; public static final String WORKFLOW_ID = "mapreduce.workflow.id"; Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1529077&r1=1529076&r2=1529077&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Fri Oct 4 05:37:54 2013 @@ -1026,9 +1026,18 @@ <description>CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. If mapreduce.application.framework is set then this must specify the appropriate classpath for that archive, and the name of - the archive must be present in the classpath.</description> + the archive must be present in the classpath. + When this value is empty, the following default CLASSPATH for MR + applications would be used. + For Linux: + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*, + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*. + For Windows: + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*, + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*. + </description> <name>mapreduce.application.classpath</name> - <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value> + <value></value> </property> <property>