Author: szetszwo Date: Wed Oct 24 20:42:03 2012 New Revision: 1401869 URL: http://svn.apache.org/viewvc?rev=1401869&view=rev Log: Merge r1400738 through r1401868 from trunk.
Added: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java - copied unchanged from r1401868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/ - copied from r1401868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml - copied unchanged from r1401868, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml Removed: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Wed Oct 24 20:42:03 2012 @@ -86,7 +86,13 @@ Release 2.0.3-alpha - Unreleased YARN-150. Fixes AppRejectedTransition does not unregister a rejected app-attempt from the ApplicationMasterService (Bikas Saha via sseth) - YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu) + YARN-140. Add capacity-scheduler-default.xml to provide a default set of + configurations for the capacity scheduler. (ahmed via tucu) + + YARN-179. Fix some unit test failures. (Vinod Kumar Vavilapalli via sseth) + + YARN-181. Fixed eclipse settings broken by capacity-scheduler.xml move via + YARN-140. (Siddharth Seth via vinodkv) Release 2.0.2-alpha - 2012-09-07 @@ -165,6 +171,20 @@ Release 0.23.5 - UNRELEASED YARN-163. Retrieving container log via NM webapp can hang with multibyte characters in log (jlowe via bobby) + YARN-174. Modify NodeManager to pass the user's configuration even when + rebooting. (vinodkv) + + YARN-177. CapacityScheduler - adding a queue while the RM is running has + wacky results (acmurthy vai tgraves) + + YARN-178. Fix custom ProcessTree instance creation (Radim Kolar via bobby) + + YARN-180. Capacity scheduler - containers that get reserved create + container token to early (acmurthy and bobby) + + YARN-139. Interrupted Exception within AsyncDispatcher leads to user + confusion. (Vinod Kumar Vavilapalli via jlowe) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml Wed Oct 24 20:42:03 2012 @@ -53,6 +53,12 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> <scope>test</scope> </dependency> Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java Wed Oct 24 20:42:03 2012 @@ -143,7 +143,7 @@ public class UnmanagedAMLauncher { appName = cliParser.getOptionValue("appname", "UnmanagedAM"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); - amQueue = cliParser.getOptionValue("queue", ""); + amQueue = cliParser.getOptionValue("queue", "default"); classpath = cliParser.getOptionValue("classpath", null); amCmd = cliParser.getOptionValue("cmd"); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java Wed Oct 24 20:42:03 2012 @@ -141,6 +141,8 @@ public class TestUnmanagedAMLauncher { String[] args = { "--classpath", classpath, + "--queue", + "default", "--cmd", javaHome + "/bin/java -Xmx512m " Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Wed Oct 24 20:42:03 2012 @@ -38,5 +38,12 @@ <artifactId>hadoop-yarn-server-resourcemanager</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + </dependencies> </project> Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Wed Oct 24 20:42:03 2012 @@ -68,7 +68,9 @@ public class AsyncDispatcher extends Abs try { event = eventQueue.take(); } catch(InterruptedException ie) { - LOG.warn("AsyncDispatcher thread interrupted", ie); + if (!stopped) { + LOG.warn("AsyncDispatcher thread interrupted", ie); + } return; } if (event != null) { @@ -180,7 +182,9 @@ public class AsyncDispatcher extends Abs try { eventQueue.put(event); } catch (InterruptedException e) { - LOG.warn("AsyncDispatcher thread interrupted", e); + if (!stopped) { + LOG.warn("AsyncDispatcher thread interrupted", e); + } throw new YarnException(e); } }; Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Wed Oct 24 20:42:03 2012 @@ -114,6 +114,7 @@ public class ProcfsBasedProcessTree exte * @param procfsDir the root of a proc file system - only used for testing. */ public ProcfsBasedProcessTree(String pid, String procfsDir) { + super(pid); this.pid = getValidPID(pid); this.procfsDir = procfsDir; } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java Wed Oct 24 20:42:03 2012 @@ -21,17 +21,28 @@ package org.apache.hadoop.yarn.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.ReflectionUtils; +import java.lang.reflect.Constructor; /** * Interface class to obtain process resource usage * */ -public abstract class ResourceCalculatorProcessTree { +public abstract class ResourceCalculatorProcessTree extends Configured { static final Log LOG = LogFactory .getLog(ResourceCalculatorProcessTree.class); /** + * Create process-tree instance with specified root process. + * + * Subclass must override this. + * @param root process-tree root-process + */ + public ResourceCalculatorProcessTree(String root) { + } + + /** * Get the process-tree with latest state. If the root-process is not alive, * an empty tree will be returned. * @@ -122,10 +133,17 @@ public abstract class ResourceCalculator * is not available for this system. */ public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree( - String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) { + String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) { if (clazz != null) { - return ReflectionUtils.newInstance(clazz, conf); + try { + Constructor <? extends ResourceCalculatorProcessTree> c = clazz.getConstructor(String.class); + ResourceCalculatorProcessTree rctree = c.newInstance(pid); + rctree.setConf(conf); + return rctree; + } catch(Exception e) { + throw new RuntimeException(e); + } } // No class given, try a os specific class Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Wed Oct 24 20:42:03 2012 @@ -253,12 +253,12 @@ public class NodeManager extends Composi if (hasToReboot) { LOG.info("Rebooting the node manager."); NodeManager nodeManager = createNewNodeManager(); - nodeManager.initAndStartNodeManager(hasToReboot); + nodeManager.initAndStartNodeManager(this.getConfig(), hasToReboot); } } } - private void initAndStartNodeManager(boolean hasToReboot) { + private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) { try { // Remove the old hook if we are rebooting. @@ -270,7 +270,6 @@ public class NodeManager extends Composi ShutdownHookManager.get().addShutdownHook(nodeManagerShutdownHook, SHUTDOWN_HOOK_PRIORITY); - YarnConfiguration conf = new YarnConfiguration(); this.init(conf); this.start(); } catch (Throwable t) { @@ -288,6 +287,7 @@ public class NodeManager extends Composi Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); NodeManager nodeManager = new NodeManager(); - nodeManager.initAndStartNodeManager(false); + Configuration conf = new YarnConfiguration(); + nodeManager.initAndStartNodeManager(conf, false); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Wed Oct 24 20:42:03 2012 @@ -50,7 +50,7 @@ <directory>${basedir}/src/test/resources</directory> </testResource> <testResource> - <directory>${basedir}/../../conf</directory> + <directory>${basedir}/conf</directory> <includes> <include>capacity-scheduler.xml</include> </includes> Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Wed Oct 24 20:42:03 2012 @@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re public CSQueue getParent(); /** + * Set the parent <code>Queue</code>. + * @param newParentQueue new parent queue + */ + public void setParent(CSQueue newParentQueue); + + /** * Get the queue name. * @return the queue name */ @@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re /** * Reinitialize the queue. - * @param queue new queue to re-initalize from + * @param newlyParsedQueue new queue to re-initalize from * @param clusterResource resources in the cluster */ - public void reinitialize(CSQueue queue, Resource clusterResource) + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException; /** Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed Oct 24 20:42:03 2012 @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; @@ -111,21 +110,18 @@ implements ResourceScheduler, CapacitySc } }; + @Override public void setConf(Configuration conf) { - if (conf instanceof YarnConfiguration) { - yarnConf = (YarnConfiguration) conf; - } else { - throw new IllegalArgumentException("Can only configure with " + - "YarnConfiguration"); - } + yarnConf = conf; } + @Override public Configuration getConf() { return yarnConf; } private CapacitySchedulerConfiguration conf; - private YarnConfiguration yarnConf; + private Configuration yarnConf; private RMContext rmContext; private Map<String, CSQueue> queues = new ConcurrentHashMap<String, CSQueue>(); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Oct 24 20:42:03 2012 @@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); - float absCapacity = parent.getAbsoluteCapacity() * capacity; + float absCapacity = getParent().getAbsoluteCapacity() * capacity; CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absCapacity, absoluteMaxCapacity); this.capacity = capacity; @@ -256,7 +256,7 @@ public class LeafQueue implements CSQueu // Update metrics CSQueueUtils.updateQueueStatistics( - this, parent, clusterResource, minimumAllocation); + this, getParent(), clusterResource, minimumAllocation); LOG.info("Initializing " + queueName + "\n" + "capacity = " + capacity + @@ -339,10 +339,15 @@ public class LeafQueue implements CSQueu } @Override - public CSQueue getParent() { + public synchronized CSQueue getParent() { return parent; } - + + @Override + public synchronized void setParent(CSQueue newParentQueue) { + this.parent = (ParentQueue)newParentQueue; + } + @Override public String getQueueName() { return queueName; @@ -350,7 +355,7 @@ public class LeafQueue implements CSQueu @Override public String getQueuePath() { - return parent.getQueuePath() + "." + getQueueName(); + return getParent().getQueuePath() + "." + getQueueName(); } /** @@ -430,7 +435,9 @@ public class LeafQueue implements CSQueu synchronized void setMaxCapacity(float maximumCapacity) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); - float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent); + float absMaxCapacity = + CSQueueUtils.computeAbsoluteMaximumCapacity( + maximumCapacity, getParent()); CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity); this.maximumCapacity = maximumCapacity; @@ -453,10 +460,6 @@ public class LeafQueue implements CSQueu this.userLimitFactor = userLimitFactor; } - synchronized void setParentQueue(CSQueue parent) { - this.parent = parent; - } - @Override public synchronized int getNumApplications() { return getNumPendingApplications() + getNumActiveApplications(); @@ -559,26 +562,28 @@ public class LeafQueue implements CSQueu } @Override - public synchronized void reinitialize(CSQueue queue, Resource clusterResource) + public synchronized void reinitialize( + CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check - if (!(queue instanceof LeafQueue) || - !queue.getQueuePath().equals(getQueuePath())) { + if (!(newlyParsedQueue instanceof LeafQueue) || + !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + - " from " + queue.getQueuePath()); + " from " + newlyParsedQueue.getQueuePath()); } - LeafQueue leafQueue = (LeafQueue)queue; + LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue; setupQueueConfigs( clusterResource, - leafQueue.capacity, leafQueue.absoluteCapacity, - leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity, - leafQueue.userLimit, leafQueue.userLimitFactor, - leafQueue.maxApplications, - leafQueue.getMaxApplicationsPerUser(), - leafQueue.getMaximumActiveApplications(), - leafQueue.getMaximumActiveApplicationsPerUser(), - leafQueue.state, leafQueue.acls); + newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity, + newlyParsedLeafQueue.maximumCapacity, + newlyParsedLeafQueue.absoluteMaxCapacity, + newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor, + newlyParsedLeafQueue.maxApplications, + newlyParsedLeafQueue.getMaxApplicationsPerUser(), + newlyParsedLeafQueue.getMaximumActiveApplications(), + newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(), + newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls); } @Override @@ -591,7 +596,7 @@ public class LeafQueue implements CSQueu } // Check if parent-queue allows access - return parent.hasAccess(acl, user); + return getParent().hasAccess(acl, user); } @Override @@ -649,10 +654,10 @@ public class LeafQueue implements CSQueu // Inform the parent queue try { - parent.submitApplication(application, userName, queue); + getParent().submitApplication(application, userName, queue); } catch (AccessControlException ace) { LOG.info("Failed to submit application to parent-queue: " + - parent.getQueuePath(), ace); + getParent().getQueuePath(), ace); removeApplication(application, user); throw ace; } @@ -708,7 +713,7 @@ public class LeafQueue implements CSQueu } // Inform the parent queue - parent.finishApplication(application, queue); + getParent().finishApplication(application, queue); } public synchronized void removeApplication(FiCaSchedulerApp application, User user) { @@ -1183,34 +1188,32 @@ public class LeafQueue implements CSQueu return (rmContainer != null) ? rmContainer.getContainer() : createContainer(application, node, capability, priority); } - - public Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, + Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, Resource capability, Priority priority) { - + NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); - ContainerToken containerToken = null; - - // If security is enabled, send the container-tokens too. - if (UserGroupInformation.isSecurityEnabled()) { - containerToken = - containerTokenSecretManager.createContainerToken(containerId, nodeId, - application.getUser(), capability); - if (containerToken == null) { - return null; // Try again later. - } - } - + // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode().getHttpAddress(), capability, priority, - containerToken); - + null); + return container; } - + + /** + * Create <code>ContainerToken</code>, only in secure-mode + */ + ContainerToken createContainerToken( + FiCaSchedulerApp application, Container container) { + return containerTokenSecretManager.createContainerToken( + container.getId(), container.getNodeId(), + application.getUser(), container.getResource()); + } + private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, ResourceRequest request, NodeType type, RMContainer rmContainer) { @@ -1246,6 +1249,17 @@ public class LeafQueue implements CSQueu unreserve(application, priority, node, rmContainer); } + // Create container tokens in secure-mode + if (UserGroupInformation.isSecurityEnabled()) { + ContainerToken containerToken = + createContainerToken(application, container); + if (containerToken == null) { + // Something went wrong... + return Resources.none(); + } + container.setContainerToken(containerToken); + } + // Inform the application RMContainer allocatedContainer = application.allocate(type, node, priority, request, container); @@ -1351,7 +1365,7 @@ public class LeafQueue implements CSQueu } // Inform the parent queue - parent.completedContainer(clusterResource, application, + getParent().completedContainer(clusterResource, application, node, rmContainer, null, event); } } @@ -1361,7 +1375,7 @@ public class LeafQueue implements CSQueu // Update queue metrics Resources.addTo(usedResources, resource); CSQueueUtils.updateQueueStatistics( - this, parent, clusterResource, minimumAllocation); + this, getParent(), clusterResource, minimumAllocation); ++numContainers; // Update user metrics @@ -1386,7 +1400,7 @@ public class LeafQueue implements CSQueu // Update queue metrics Resources.subtractFrom(usedResources, resource); CSQueueUtils.updateQueueStatistics( - this, parent, clusterResource, minimumAllocation); + this, getParent(), clusterResource, minimumAllocation); --numContainers; // Update user metrics @@ -1417,7 +1431,7 @@ public class LeafQueue implements CSQueu // Update metrics CSQueueUtils.updateQueueStatistics( - this, parent, clusterResource, minimumAllocation); + this, getParent(), clusterResource, minimumAllocation); // Update application properties for (FiCaSchedulerApp application : activeApplications) { @@ -1488,7 +1502,7 @@ public class LeafQueue implements CSQueu synchronized (this) { allocateResource(clusterResource, application, container.getResource()); } - parent.recoverContainer(clusterResource, application, container); + getParent().recoverContainer(clusterResource, application, container); } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Wed Oct 24 20:42:03 2012 @@ -60,7 +60,7 @@ public class ParentQueue implements CSQu private static final Log LOG = LogFactory.getLog(ParentQueue.class); - private final CSQueue parent; + private CSQueue parent; private final String queueName; private float capacity; @@ -216,11 +216,16 @@ public class ParentQueue implements CSQu } @Override - public CSQueue getParent() { + public synchronized CSQueue getParent() { return parent; } @Override + public synchronized void setParent(CSQueue newParentQueue) { + this.parent = (ParentQueue)newParentQueue; + } + + @Override public String getQueueName() { return queueName; } @@ -357,37 +362,52 @@ public class ParentQueue implements CSQu } @Override - public synchronized void reinitialize(CSQueue queue, Resource clusterResource) + public synchronized void reinitialize( + CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { // Sanity check - if (!(queue instanceof ParentQueue) || - !queue.getQueuePath().equals(getQueuePath())) { + if (!(newlyParsedQueue instanceof ParentQueue) || + !newlyParsedQueue.getQueuePath().equals(getQueuePath())) { throw new IOException("Trying to reinitialize " + getQueuePath() + - " from " + queue.getQueuePath()); + " from " + newlyParsedQueue.getQueuePath()); } - ParentQueue parentQueue = (ParentQueue)queue; + ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue; // Set new configs setupQueueConfigs(clusterResource, - parentQueue.capacity, parentQueue.absoluteCapacity, - parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity, - parentQueue.state, parentQueue.acls); + newlyParsedParentQueue.capacity, + newlyParsedParentQueue.absoluteCapacity, + newlyParsedParentQueue.maximumCapacity, + newlyParsedParentQueue.absoluteMaxCapacity, + newlyParsedParentQueue.state, + newlyParsedParentQueue.acls); // Re-configure existing child queues and add new ones // The CS has already checked to ensure all existing child queues are present! Map<String, CSQueue> currentChildQueues = getQueues(childQueues); - Map<String, CSQueue> newChildQueues = getQueues(parentQueue.childQueues); + Map<String, CSQueue> newChildQueues = + getQueues(newlyParsedParentQueue.childQueues); for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) { String newChildQueueName = e.getKey(); CSQueue newChildQueue = e.getValue(); CSQueue childQueue = currentChildQueues.get(newChildQueueName); - if (childQueue != null){ + + // Check if the child-queue already exists + if (childQueue != null) { + // Re-init existing child queues childQueue.reinitialize(newChildQueue, clusterResource); LOG.info(getQueueName() + ": re-configured queue: " + childQueue); } else { + // New child queue, do not re-init + + // Set parent to 'this' + newChildQueue.setParent(this); + + // Save in list of current child queues currentChildQueues.put(newChildQueueName, newChildQueue); + LOG.info(getQueueName() + ": added new child queue: " + newChildQueue); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Wed Oct 24 20:42:03 2012 @@ -378,4 +378,43 @@ public class TestCapacityScheduler { Assert.assertEquals(4 * GB, cs.getClusterResources().getMemory()); } + + @Test + public void testRefreshQueuesWithNewQueue() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + + // Add a new queue b4 + String B4 = B + ".b4"; + float B4_CAPACITY = 10; + + B3_CAPACITY -= B4_CAPACITY; + try { + conf.setCapacity(A, 80f); + conf.setCapacity(B, 20f); + conf.setQueues(B, new String[] {"b1", "b2", "b3", "b4"}); + conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B4, B4_CAPACITY); + cs.reinitialize(conf,null); + checkQueueCapacities(cs, 80f, 20f); + + // Verify parent for B4 + CSQueue rootQueue = cs.getRootQueue(); + CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueB4 = findQueue(queueB, B4); + + assertEquals(queueB, queueB4.getParent()); + } finally { + B3_CAPACITY += B4_CAPACITY; + } + } + }