Author: jlowe Date: Wed Nov 14 00:06:01 2012 New Revision: 1409041 URL: http://svn.apache.org/viewvc?rev=1409041&view=rev Log: YARN-144. MiniMRYarnCluster launches RM and JHS on default ports. Contributed by Robert Parker
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Wed Nov 14 00:06:01 2012 @@ -83,6 +83,9 @@ Release 0.23.5 - UNRELEASED YARN-212. NM state machine ignores an APPLICATION_CONTAINER_FINISHED event when it shouldn't (Nathan Roberts via jlowe) + YARN-144. MiniMRYarnCluster launches RM and JHS on default ports (Robert + Parker via jlowe) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Wed Nov 14 00:06:01 2012 @@ -76,7 +76,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; @@ -198,13 +197,19 @@ public class Client { /** */ - public Client() throws Exception { + public Client(Configuration conf) throws Exception { // Set up the configuration and RPC - conf = new Configuration(); + this.conf = conf; rpc = YarnRPC.create(conf); } /** + */ + public Client() throws Exception { + this(new Configuration()); + } + + /** * Helper function to print out usage * @param opts Parsed command line options */ @@ -504,22 +509,20 @@ public class Client { // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar - String classPathEnv = "${CLASSPATH}" - + ":./*" - + ":$HADOOP_CONF_DIR" - + ":$HADOOP_COMMON_HOME/share/hadoop/common/*" - + ":$HADOOP_COMMON_HOME/share/hadoop/common/lib/*" - + ":$HADOOP_HDFS_HOME/share/hadoop/hdfs/*" - + ":$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*" - + ":$YARN_HOME/modules/*" - + ":$YARN_HOME/lib/*" - + ":./log4j.properties:"; + StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*"); + for (String c : conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + .split(",")) { + classPathEnv.append(':'); + classPathEnv.append(c.trim()); + } + classPathEnv.append(":./log4j.properties"); - // add the runtime classpath needed for tests to work + // add the runtime classpath needed for tests to work String testRuntimeClassPath = Client.getTestRuntimeClasspath(); - classPathEnv += ":" + testRuntimeClassPath; + classPathEnv.append(':'); + classPathEnv.append(testRuntimeClassPath); - env.put("CLASSPATH", classPathEnv); + env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Wed Nov 14 00:06:01 2012 @@ -18,11 +18,17 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; @@ -53,6 +59,14 @@ public class TestDistributedShell { 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); + URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); + } + yarnCluster.getConfig().set("yarn.application.classpath", new File(url.getPath()).getParent()); + OutputStream os = new FileOutputStream(new File(url.getPath())); + yarnCluster.getConfig().writeXml(os); + os.close(); } try { Thread.sleep(2000); @@ -86,14 +100,14 @@ public class TestDistributedShell { }; LOG.info("Initializing DS Client"); - Client client = new Client(); + Client client = new Client(new Configuration(yarnCluster.getConfig())); boolean initSuccess = client.init(args); - assert(initSuccess); + Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = client.run(); LOG.info("Client run completed. Result=" + result); - assert (result == true); + Assert.assertTrue(result); } Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml?rev=1409041&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml (added) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml Wed Nov 14 00:06:01 2012 @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<configuration> + <!-- Dummy (invalid) config file to be overwriten by TestDistributedShell with MiniCluster configuration. --> +</configuration> + + Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Nov 14 00:06:01 2012 @@ -567,6 +567,18 @@ public class YarnConfiguration extends C /** Container temp directory */ public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; + public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + ".is.minicluster"; + + /** Whether to use fixed ports with the minicluster. */ + public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX + + "minicluster.fixed.ports"; + + /** + * Default is false to be able to run tests concurrently without port + * conflicts. + */ + public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; + public YarnConfiguration() { super(); } Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Nov 14 00:06:01 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re import java.io.IOException; +import java.net.InetAddress; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -506,6 +507,15 @@ public class ResourceManager extends Com } catch(IOException ie) { throw new YarnException("Failed to start secret manager threads", ie); } + + if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); + hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname; + int port = webApp.port(); + String resolvedAddress = hostname + ":" + port; + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); + } super.start(); Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1409041&r1=1409040&r2=1409041&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Wed Nov 14 00:06:01 2012 @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server; import java.io.File; import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.UnknownHostException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -118,7 +121,16 @@ public class MiniYARNCluster extends Com public NodeManager getNodeManager(int i) { return this.nodeManagers[i]; } - + + public static String getHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException ex) { + throw new RuntimeException(ex); + } + } + private class ResourceManagerWrapper extends AbstractService { public ResourceManagerWrapper() { super(ResourceManagerWrapper.class.getName()); @@ -127,6 +139,22 @@ public class MiniYARNCluster extends Com @Override public synchronized void start() { try { + getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); + if (!getConfig().getBoolean( + YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, + YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) { + // pick free random ports. + getConfig().set(YarnConfiguration.RM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + } Store store = StoreFactory.getStore(getConfig()); resourceManager = new ResourceManager(store) { @Override @@ -156,6 +184,10 @@ public class MiniYARNCluster extends Com } catch (Throwable t) { throw new YarnException(t); } + LOG.info("MiniYARN ResourceManager address: " + + getConfig().get(YarnConfiguration.RM_ADDRESS)); + LOG.info("MiniYARN ResourceManager web address: " + + getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); } @Override @@ -217,9 +249,12 @@ public class MiniYARNCluster extends Com remoteLogDir.getAbsolutePath()); // By default AM + 2 containers getConfig().setInt(YarnConfiguration.NM_PMEM_MB, 4*1024); - getConfig().set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:0"); - getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:0"); - getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, "0.0.0.0:0"); + getConfig().set(YarnConfiguration.NM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); LOG.info("Starting NM: " + index); nodeManagers[index].init(getConfig()); new Thread() {