Author: tomwhite Date: Mon Nov 19 15:12:22 2012 New Revision: 1411235 URL: http://svn.apache.org/viewvc?rev=1411235&view=rev Log: YARN-129. Simplify classpath construction for mini YARN tests.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411235&r1=1411234&r2=1411235&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Mon Nov 19 15:12:22 2012 @@ -89,11 +89,6 @@ <phase>test-compile</phase> </execution> </executions> - <configuration> - <excludes> - <exclude>mrapp-generated-classpath</exclude> - </excludes> - </configuration> </plugin> <plugin> <artifactId>maven-antrun-plugin</artifactId> Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1411235&r1=1411234&r2=1411235&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Mon Nov 19 15:12:22 2012 @@ -18,13 +18,8 @@ package org.apache.hadoop.mapreduce.v2.util; -import java.io.BufferedReader; -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URI; -import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -134,62 +129,24 @@ public class MRApps extends Apps { private static void setMRFrameworkClasspath( Map<String, String> environment, Configuration conf) throws IOException { - InputStream classpathFileStream = null; - BufferedReader reader = null; - try { - // Get yarn mapreduce-app classpath from generated classpath - // Works if compile time env is same as runtime. Mainly tests. - ClassLoader thisClassLoader = - Thread.currentThread().getContextClassLoader(); - String mrAppGeneratedClasspathFile = "mrapp-generated-classpath"; - classpathFileStream = - thisClassLoader.getResourceAsStream(mrAppGeneratedClasspathFile); - - // Put the file itself on classpath for tasks. - URL classpathResource = thisClassLoader - .getResource(mrAppGeneratedClasspathFile); - if (classpathResource != null) { - String classpathElement = classpathResource.getFile(); - if (classpathElement.contains("!")) { - classpathElement = classpathElement.substring(0, - classpathElement.indexOf("!")); - } else { - classpathElement = new File(classpathElement).getParent(); - } - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), - classpathElement); - } - - if (classpathFileStream != null) { - reader = new BufferedReader(new InputStreamReader(classpathFileStream, - Charsets.UTF_8)); - String cp = reader.readLine(); - if (cp != null) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), - cp.trim()); - } - } - - // Add standard Hadoop classes - for (String c : conf.getStrings( - YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c - .trim()); - } - for (String c : conf.getStrings( - MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, - MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c - .trim()); - } - } finally { - if (classpathFileStream != null) { - classpathFileStream.close(); - } - if (reader != null) { - reader.close(); - } + // Propagate the system classpath when using the mini cluster + if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), + System.getProperty("java.class.path")); + } + + // Add standard Hadoop classes + for (String c : conf.getStrings( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c + .trim()); + } + for (String c : conf.getStrings( + MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c + .trim()); } // TODO: Remove duplicates. } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java?rev=1411235&r1=1411234&r2=1411235&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Mon Nov 19 15:12:22 2012 @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJ import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -52,6 +53,8 @@ public class TestEncryptedShuffle { private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" + TestEncryptedShuffle.class.getSimpleName(); + + private String classpathDir; @BeforeClass public static void setUp() throws Exception { @@ -62,27 +65,12 @@ public class TestEncryptedShuffle { @Before public void createCustomYarnClasspath() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - - URL url = Thread.currentThread().getContextClassLoader(). - getResource("mrapp-generated-classpath"); - File f = new File(url.getPath()); - BufferedReader reader = new BufferedReader(new FileReader(f)); - String cp = reader.readLine(); - cp = cp + ":" + classpathDir; - f = new File(classpathDir, "mrapp-generated-classpath"); - Writer writer = new FileWriter(f); - writer.write(cp); - writer.close(); + classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); new File(classpathDir, "core-site.xml").delete(); } @After public void cleanUpMiniClusterSpecialConfig() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - new File(classpathDir, "mrapp-generated-classpath").delete(); new File(classpathDir, "core-site.xml").delete(); String keystoresDir = new File(BASEDIR).getAbsolutePath(); KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir); @@ -98,6 +86,9 @@ public class TestEncryptedShuffle { conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); + String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + + File.pathSeparator + classpathDir; + conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); dfsCluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); @@ -113,8 +104,6 @@ public class TestEncryptedShuffle { mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); // so the minicluster conf is avail to the containers. - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); Writer writer = new FileWriter(classpathDir + "/core-site.xml"); mrCluster.getConfig().writeXml(writer); writer.close();