Author: szetszwo
Date: Wed Oct 24 20:42:03 2012
New Revision: 1401869
URL: http://svn.apache.org/viewvc?rev=1401869&view=rev
Log:
Merge r1400738 through r1401868 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
- copied unchanged from r1401868,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/
- copied from r1401868,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
- copied unchanged from r1401868,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Wed Oct 24
20:42:03 2012
@@ -86,7 +86,13 @@ Release 2.0.3-alpha - Unreleased
YARN-150. Fixes AppRejectedTransition does not unregister a rejected
app-attempt from the ApplicationMasterService (Bikas Saha via sseth)
- YARN-140. Add capacity-scheduler-default.xml to provide a default set of
configurations for the capacity scheduler. (ahmed via tucu)
+ YARN-140. Add capacity-scheduler-default.xml to provide a default set of
+ configurations for the capacity scheduler. (ahmed via tucu)
+
+ YARN-179. Fix some unit test failures. (Vinod Kumar Vavilapalli via sseth)
+
+ YARN-181. Fixed eclipse settings broken by capacity-scheduler.xml move via
+ YARN-140. (Siddharth Seth via vinodkv)
Release 2.0.2-alpha - 2012-09-07
@@ -165,6 +171,20 @@ Release 0.23.5 - UNRELEASED
YARN-163. Retrieving container log via NM webapp can hang with multibyte
characters in log (jlowe via bobby)
+ YARN-174. Modify NodeManager to pass the user's configuration even when
+ rebooting. (vinodkv)
+
+ YARN-177. CapacityScheduler - adding a queue while the RM is running has
+ wacky results (acmurthy vai tgraves)
+
+ YARN-178. Fix custom ProcessTree instance creation (Radim Kolar via bobby)
+
+ YARN-180. Capacity scheduler - containers that get reserved create
+ container token to early (acmurthy and bobby)
+
+ YARN-139. Interrupted Exception within AsyncDispatcher leads to user
+ confusion. (Vinod Kumar Vavilapalli via jlowe)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
Wed Oct 24 20:42:03 2012
@@ -53,6 +53,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
Wed Oct 24 20:42:03 2012
@@ -143,7 +143,7 @@ public class UnmanagedAMLauncher {
appName = cliParser.getOptionValue("appname", "UnmanagedAM");
amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
- amQueue = cliParser.getOptionValue("queue", "");
+ amQueue = cliParser.getOptionValue("queue", "default");
classpath = cliParser.getOptionValue("classpath", null);
amCmd = cliParser.getOptionValue("cmd");
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
Wed Oct 24 20:42:03 2012
@@ -141,6 +141,8 @@ public class TestUnmanagedAMLauncher {
String[] args = {
"--classpath",
classpath,
+ "--queue",
+ "default",
"--cmd",
javaHome
+ "/bin/java -Xmx512m "
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
Wed Oct 24 20:42:03 2012
@@ -38,5 +38,12 @@
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
</dependencies>
</project>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
Wed Oct 24 20:42:03 2012
@@ -68,7 +68,9 @@ public class AsyncDispatcher extends Abs
try {
event = eventQueue.take();
} catch(InterruptedException ie) {
- LOG.warn("AsyncDispatcher thread interrupted", ie);
+ if (!stopped) {
+ LOG.warn("AsyncDispatcher thread interrupted", ie);
+ }
return;
}
if (event != null) {
@@ -180,7 +182,9 @@ public class AsyncDispatcher extends Abs
try {
eventQueue.put(event);
} catch (InterruptedException e) {
- LOG.warn("AsyncDispatcher thread interrupted", e);
+ if (!stopped) {
+ LOG.warn("AsyncDispatcher thread interrupted", e);
+ }
throw new YarnException(e);
}
};
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
Wed Oct 24 20:42:03 2012
@@ -114,6 +114,7 @@ public class ProcfsBasedProcessTree exte
* @param procfsDir the root of a proc file system - only used for testing.
*/
public ProcfsBasedProcessTree(String pid, String procfsDir) {
+ super(pid);
this.pid = getValidPID(pid);
this.procfsDir = procfsDir;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
Wed Oct 24 20:42:03 2012
@@ -21,17 +21,28 @@ package org.apache.hadoop.yarn.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.ReflectionUtils;
+import java.lang.reflect.Constructor;
/**
* Interface class to obtain process resource usage
*
*/
-public abstract class ResourceCalculatorProcessTree {
+public abstract class ResourceCalculatorProcessTree extends Configured {
static final Log LOG = LogFactory
.getLog(ResourceCalculatorProcessTree.class);
/**
+ * Create process-tree instance with specified root process.
+ *
+ * Subclass must override this.
+ * @param root process-tree root-process
+ */
+ public ResourceCalculatorProcessTree(String root) {
+ }
+
+ /**
* Get the process-tree with latest state. If the root-process is not alive,
* an empty tree will be returned.
*
@@ -122,10 +133,17 @@ public abstract class ResourceCalculator
* is not available for this system.
*/
public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree(
- String pid, Class<? extends ResourceCalculatorProcessTree> clazz,
Configuration conf) {
+ String pid, Class<? extends ResourceCalculatorProcessTree> clazz,
Configuration conf) {
if (clazz != null) {
- return ReflectionUtils.newInstance(clazz, conf);
+ try {
+ Constructor <? extends ResourceCalculatorProcessTree> c =
clazz.getConstructor(String.class);
+ ResourceCalculatorProcessTree rctree = c.newInstance(pid);
+ rctree.setConf(conf);
+ return rctree;
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
}
// No class given, try a os specific class
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
Wed Oct 24 20:42:03 2012
@@ -253,12 +253,12 @@ public class NodeManager extends Composi
if (hasToReboot) {
LOG.info("Rebooting the node manager.");
NodeManager nodeManager = createNewNodeManager();
- nodeManager.initAndStartNodeManager(hasToReboot);
+ nodeManager.initAndStartNodeManager(this.getConfig(), hasToReboot);
}
}
}
- private void initAndStartNodeManager(boolean hasToReboot) {
+ private void initAndStartNodeManager(Configuration conf, boolean
hasToReboot) {
try {
// Remove the old hook if we are rebooting.
@@ -270,7 +270,6 @@ public class NodeManager extends Composi
ShutdownHookManager.get().addShutdownHook(nodeManagerShutdownHook,
SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
this.init(conf);
this.start();
} catch (Throwable t) {
@@ -288,6 +287,7 @@ public class NodeManager extends Composi
Thread.setDefaultUncaughtExceptionHandler(new
YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
NodeManager nodeManager = new NodeManager();
- nodeManager.initAndStartNodeManager(false);
+ Configuration conf = new YarnConfiguration();
+ nodeManager.initAndStartNodeManager(conf, false);
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
Wed Oct 24 20:42:03 2012
@@ -50,7 +50,7 @@
<directory>${basedir}/src/test/resources</directory>
</testResource>
<testResource>
- <directory>${basedir}/../../conf</directory>
+ <directory>${basedir}/conf</directory>
<includes>
<include>capacity-scheduler.xml</include>
</includes>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
Wed Oct 24 20:42:03 2012
@@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re
public CSQueue getParent();
/**
+ * Set the parent <code>Queue</code>.
+ * @param newParentQueue new parent queue
+ */
+ public void setParent(CSQueue newParentQueue);
+
+ /**
* Get the queue name.
* @return the queue name
*/
@@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re
/**
* Reinitialize the queue.
- * @param queue new queue to re-initalize from
+ * @param newlyParsedQueue new queue to re-initalize from
* @param clusterResource resources in the cluster
*/
- public void reinitialize(CSQueue queue, Resource clusterResource)
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException;
/**
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Wed Oct 24 20:42:03 2012
@@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import
org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -111,21 +110,18 @@ implements ResourceScheduler, CapacitySc
}
};
+ @Override
public void setConf(Configuration conf) {
- if (conf instanceof YarnConfiguration) {
- yarnConf = (YarnConfiguration) conf;
- } else {
- throw new IllegalArgumentException("Can only configure with " +
- "YarnConfiguration");
- }
+ yarnConf = conf;
}
+ @Override
public Configuration getConf() {
return yarnConf;
}
private CapacitySchedulerConfiguration conf;
- private YarnConfiguration yarnConf;
+ private Configuration yarnConf;
private RMContext rmContext;
private Map<String, CSQueue> queues = new ConcurrentHashMap<String,
CSQueue>();
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Wed Oct 24 20:42:03 2012
@@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absCapacity = parent.getAbsoluteCapacity() * capacity;
+ float absCapacity = getParent().getAbsoluteCapacity() * capacity;
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absCapacity,
absoluteMaxCapacity);
this.capacity = capacity;
@@ -256,7 +256,7 @@ public class LeafQueue implements CSQueu
// Update metrics
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + capacity +
@@ -339,10 +339,15 @@ public class LeafQueue implements CSQueu
}
@Override
- public CSQueue getParent() {
+ public synchronized CSQueue getParent() {
return parent;
}
-
+
+ @Override
+ public synchronized void setParent(CSQueue newParentQueue) {
+ this.parent = (ParentQueue)newParentQueue;
+ }
+
@Override
public String getQueueName() {
return queueName;
@@ -350,7 +355,7 @@ public class LeafQueue implements CSQueu
@Override
public String getQueuePath() {
- return parent.getQueuePath() + "." + getQueueName();
+ return getParent().getQueuePath() + "." + getQueueName();
}
/**
@@ -430,7 +435,9 @@ public class LeafQueue implements CSQueu
synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absMaxCapacity =
CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
+ float absMaxCapacity =
+ CSQueueUtils.computeAbsoluteMaximumCapacity(
+ maximumCapacity, getParent());
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity,
absMaxCapacity);
this.maximumCapacity = maximumCapacity;
@@ -453,10 +460,6 @@ public class LeafQueue implements CSQueu
this.userLimitFactor = userLimitFactor;
}
- synchronized void setParentQueue(CSQueue parent) {
- this.parent = parent;
- }
-
@Override
public synchronized int getNumApplications() {
return getNumPendingApplications() + getNumActiveApplications();
@@ -559,26 +562,28 @@ public class LeafQueue implements CSQueu
}
@Override
- public synchronized void reinitialize(CSQueue queue, Resource
clusterResource)
+ public synchronized void reinitialize(
+ CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
// Sanity check
- if (!(queue instanceof LeafQueue) ||
- !queue.getQueuePath().equals(getQueuePath())) {
+ if (!(newlyParsedQueue instanceof LeafQueue) ||
+ !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + queue.getQueuePath());
+ " from " + newlyParsedQueue.getQueuePath());
}
- LeafQueue leafQueue = (LeafQueue)queue;
+ LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
setupQueueConfigs(
clusterResource,
- leafQueue.capacity, leafQueue.absoluteCapacity,
- leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity,
- leafQueue.userLimit, leafQueue.userLimitFactor,
- leafQueue.maxApplications,
- leafQueue.getMaxApplicationsPerUser(),
- leafQueue.getMaximumActiveApplications(),
- leafQueue.getMaximumActiveApplicationsPerUser(),
- leafQueue.state, leafQueue.acls);
+ newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity,
+ newlyParsedLeafQueue.maximumCapacity,
+ newlyParsedLeafQueue.absoluteMaxCapacity,
+ newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor,
+ newlyParsedLeafQueue.maxApplications,
+ newlyParsedLeafQueue.getMaxApplicationsPerUser(),
+ newlyParsedLeafQueue.getMaximumActiveApplications(),
+ newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
+ newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
}
@Override
@@ -591,7 +596,7 @@ public class LeafQueue implements CSQueu
}
// Check if parent-queue allows access
- return parent.hasAccess(acl, user);
+ return getParent().hasAccess(acl, user);
}
@Override
@@ -649,10 +654,10 @@ public class LeafQueue implements CSQueu
// Inform the parent queue
try {
- parent.submitApplication(application, userName, queue);
+ getParent().submitApplication(application, userName, queue);
} catch (AccessControlException ace) {
LOG.info("Failed to submit application to parent-queue: " +
- parent.getQueuePath(), ace);
+ getParent().getQueuePath(), ace);
removeApplication(application, user);
throw ace;
}
@@ -708,7 +713,7 @@ public class LeafQueue implements CSQueu
}
// Inform the parent queue
- parent.finishApplication(application, queue);
+ getParent().finishApplication(application, queue);
}
public synchronized void removeApplication(FiCaSchedulerApp application,
User user) {
@@ -1183,34 +1188,32 @@ public class LeafQueue implements CSQueu
return (rmContainer != null) ? rmContainer.getContainer() :
createContainer(application, node, capability, priority);
}
-
- public Container createContainer(FiCaSchedulerApp application,
FiCaSchedulerNode node,
+ Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode
node,
Resource capability, Priority priority) {
-
+
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- ContainerToken containerToken = null;
-
- // If security is enabled, send the container-tokens too.
- if (UserGroupInformation.isSecurityEnabled()) {
- containerToken =
- containerTokenSecretManager.createContainerToken(containerId, nodeId,
- application.getUser(), capability);
- if (containerToken == null) {
- return null; // Try again later.
- }
- }
-
+
// Create the container
Container container = BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability, priority,
- containerToken);
-
+ null);
+
return container;
}
-
+
+ /**
+ * Create <code>ContainerToken</code>, only in secure-mode
+ */
+ ContainerToken createContainerToken(
+ FiCaSchedulerApp application, Container container) {
+ return containerTokenSecretManager.createContainerToken(
+ container.getId(), container.getNodeId(),
+ application.getUser(), container.getResource());
+ }
+
private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode
node,
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
@@ -1246,6 +1249,17 @@ public class LeafQueue implements CSQueu
unreserve(application, priority, node, rmContainer);
}
+ // Create container tokens in secure-mode
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ContainerToken containerToken =
+ createContainerToken(application, container);
+ if (containerToken == null) {
+ // Something went wrong...
+ return Resources.none();
+ }
+ container.setContainerToken(containerToken);
+ }
+
// Inform the application
RMContainer allocatedContainer =
application.allocate(type, node, priority, request, container);
@@ -1351,7 +1365,7 @@ public class LeafQueue implements CSQueu
}
// Inform the parent queue
- parent.completedContainer(clusterResource, application,
+ getParent().completedContainer(clusterResource, application,
node, rmContainer, null, event);
}
}
@@ -1361,7 +1375,7 @@ public class LeafQueue implements CSQueu
// Update queue metrics
Resources.addTo(usedResources, resource);
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
++numContainers;
// Update user metrics
@@ -1386,7 +1400,7 @@ public class LeafQueue implements CSQueu
// Update queue metrics
Resources.subtractFrom(usedResources, resource);
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
--numContainers;
// Update user metrics
@@ -1417,7 +1431,7 @@ public class LeafQueue implements CSQueu
// Update metrics
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
// Update application properties
for (FiCaSchedulerApp application : activeApplications) {
@@ -1488,7 +1502,7 @@ public class LeafQueue implements CSQueu
synchronized (this) {
allocateResource(clusterResource, application, container.getResource());
}
- parent.recoverContainer(clusterResource, application, container);
+ getParent().recoverContainer(clusterResource, application, container);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
Wed Oct 24 20:42:03 2012
@@ -60,7 +60,7 @@ public class ParentQueue implements CSQu
private static final Log LOG = LogFactory.getLog(ParentQueue.class);
- private final CSQueue parent;
+ private CSQueue parent;
private final String queueName;
private float capacity;
@@ -216,11 +216,16 @@ public class ParentQueue implements CSQu
}
@Override
- public CSQueue getParent() {
+ public synchronized CSQueue getParent() {
return parent;
}
@Override
+ public synchronized void setParent(CSQueue newParentQueue) {
+ this.parent = (ParentQueue)newParentQueue;
+ }
+
+ @Override
public String getQueueName() {
return queueName;
}
@@ -357,37 +362,52 @@ public class ParentQueue implements CSQu
}
@Override
- public synchronized void reinitialize(CSQueue queue, Resource
clusterResource)
+ public synchronized void reinitialize(
+ CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
// Sanity check
- if (!(queue instanceof ParentQueue) ||
- !queue.getQueuePath().equals(getQueuePath())) {
+ if (!(newlyParsedQueue instanceof ParentQueue) ||
+ !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + queue.getQueuePath());
+ " from " + newlyParsedQueue.getQueuePath());
}
- ParentQueue parentQueue = (ParentQueue)queue;
+ ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue;
// Set new configs
setupQueueConfigs(clusterResource,
- parentQueue.capacity, parentQueue.absoluteCapacity,
- parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity,
- parentQueue.state, parentQueue.acls);
+ newlyParsedParentQueue.capacity,
+ newlyParsedParentQueue.absoluteCapacity,
+ newlyParsedParentQueue.maximumCapacity,
+ newlyParsedParentQueue.absoluteMaxCapacity,
+ newlyParsedParentQueue.state,
+ newlyParsedParentQueue.acls);
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are
present!
Map<String, CSQueue> currentChildQueues = getQueues(childQueues);
- Map<String, CSQueue> newChildQueues = getQueues(parentQueue.childQueues);
+ Map<String, CSQueue> newChildQueues =
+ getQueues(newlyParsedParentQueue.childQueues);
for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) {
String newChildQueueName = e.getKey();
CSQueue newChildQueue = e.getValue();
CSQueue childQueue = currentChildQueues.get(newChildQueueName);
- if (childQueue != null){
+
+ // Check if the child-queue already exists
+ if (childQueue != null) {
+ // Re-init existing child queues
childQueue.reinitialize(newChildQueue, clusterResource);
LOG.info(getQueueName() + ": re-configured queue: " + childQueue);
} else {
+ // New child queue, do not re-init
+
+ // Set parent to 'this'
+ newChildQueue.setParent(this);
+
+ // Save in list of current child queues
currentChildQueues.put(newChildQueueName, newChildQueue);
+
LOG.info(getQueueName() + ": added new child queue: " + newChildQueue);
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1401869&r1=1401868&r2=1401869&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Wed Oct 24 20:42:03 2012
@@ -378,4 +378,43 @@ public class TestCapacityScheduler {
Assert.assertEquals(4 * GB, cs.getClusterResources().getMemory());
}
+
+ @Test
+ public void testRefreshQueuesWithNewQueue() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ cs.setConf(new YarnConfiguration());
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+ // Add a new queue b4
+ String B4 = B + ".b4";
+ float B4_CAPACITY = 10;
+
+ B3_CAPACITY -= B4_CAPACITY;
+ try {
+ conf.setCapacity(A, 80f);
+ conf.setCapacity(B, 20f);
+ conf.setQueues(B, new String[] {"b1", "b2", "b3", "b4"});
+ conf.setCapacity(B1, B1_CAPACITY);
+ conf.setCapacity(B2, B2_CAPACITY);
+ conf.setCapacity(B3, B3_CAPACITY);
+ conf.setCapacity(B4, B4_CAPACITY);
+ cs.reinitialize(conf,null);
+ checkQueueCapacities(cs, 80f, 20f);
+
+ // Verify parent for B4
+ CSQueue rootQueue = cs.getRootQueue();
+ CSQueue queueB = findQueue(rootQueue, B);
+ CSQueue queueB4 = findQueue(queueB, B4);
+
+ assertEquals(queueB, queueB4.getParent());
+ } finally {
+ B3_CAPACITY += B4_CAPACITY;
+ }
+ }
+
}
