Author: acmurthy
Date: Fri Sep 7 04:59:20 2012
New Revision: 1381869
URL: http://svn.apache.org/viewvc?rev=1381869&view=rev
Log:
Merge -c r1381868 from trunk to branch-2 to fix MR AM supplies MapReduce jars
in classpath rather than rely on YARN. Contributed by Arun C Murthy.
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1381869&r1=1381868&r2=1381869&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Sep 7 04:59:20 2012
@@ -111,6 +111,9 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4579. Split TestTaskAttempt into two so as to pass tests on
jdk7. (Thomas Graves via vinodkv)
+ MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than
+ rely on YARN. (acmurthy)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1381869&r1=1381868&r2=1381869&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Fri Sep 7 04:59:20 2012
@@ -179,6 +179,12 @@ public class MRApps extends Apps {
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
.trim());
}
+ for (String c : conf.getStrings(
+ MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+ MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+ .trim());
+ }
} finally {
if (classpathFileStream != null) {
classpathFileStream.close();
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1381869&r1=1381868&r2=1381869&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Fri Sep 7 04:59:20 2012
@@ -140,11 +140,19 @@ public class TestMRApps {
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
- String confClasspath =
job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
- if (confClasspath != null) {
- confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+ String yarnAppClasspath =
+ job.getConfiguration().get(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH);
+ if (yarnAppClasspath != null) {
+ yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
}
- assertTrue(environment.get("CLASSPATH").contains(confClasspath));
+ assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
+ String mrAppClasspath =
+
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
+ if (mrAppClasspath != null) {
+ mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
+ }
+ assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
@Test public void testSetClasspathWithUserPrecendence() {
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1381869&r1=1381868&r2=1381869&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Fri Sep 7 04:59:20 2012
@@ -587,4 +587,18 @@ public interface MRJobConfig {
MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
"security.job.client.protocol.acl";
+ /**
+ * CLASSPATH for all YARN MapReduce applications.
+ */
+ public static final String MAPREDUCE_APPLICATION_CLASSPATH =
+ "mapreduce.application.classpath";
+
+ /**
+ * Default CLASSPATH for all YARN MapReduce applications.
+ */
+ public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = {
+ "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*",
+ "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
+ };
+
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1381869&r1=1381868&r2=1381869&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Sep 7 04:59:20 2012
@@ -1203,6 +1203,8 @@
<!-- end of node health script variables -->
+<!-- MR YARN Application properties -->
+
<property>
<name>mapreduce.job.counters.limit</name>
<value>120</value>
@@ -1312,6 +1314,13 @@
<description>The amount of memory the MR AppMaster needs.</description>
</property>
+<property>
+ <description>CLASSPATH for MR applications. A comma-separated list
+ of CLASSPATH entries</description>
+ <name>mapreduce.application.classpath</name>
+
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+</property>
+
<!-- jobhistory properties -->
<property>
