Author: bobby
Date: Mon Jul 23 19:35:56 2012
New Revision: 1364766
URL: http://svn.apache.org/viewvc?rev=1364766&view=rev
Log:
svn merge -c 1364764 FIXES: MAPREDUCE-3893. allow capacity scheduler configs
max-apps and max-am-pct per queue (tgraves via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1364766&r1=1364765&r2=1364766&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon
Jul 23 19:35:56 2012
@@ -343,6 +343,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
init. (Jason Lowe via daryn)
+ MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and
+ maximum-am-resource-percent configurable on a per queue basis (tgraves via
+ bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1364766&r1=1364765&r2=1364766&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
Mon Jul 23 19:35:56 2012
@@ -47,12 +47,20 @@ public class CapacitySchedulerConfigurat
public static final String DOT = ".";
@Private
+ public static final String MAXIMUM_APPLICATIONS_SUFFIX =
+ "maximum-applications";
+
+ @Private
public static final String MAXIMUM_SYSTEM_APPLICATIONS =
- PREFIX + "maximum-applications";
+ PREFIX + MAXIMUM_APPLICATIONS_SUFFIX;
+
+ @Private
+ public static final String MAXIMUM_AM_RESOURCE_SUFFIX =
+ "maximum-am-resource-percent";
@Private
public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT =
- PREFIX + "maximum-am-resource-percent";
+ PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX;
@Private
public static final String QUEUES = "queues";
@@ -131,6 +139,30 @@ public class CapacitySchedulerConfigurat
return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT);
}
+
+
+ /**
+ * Get the maximum applications per queue setting.
+ * @param queue name of the queue
+ * @return setting specified or -1 if not set
+ */
+ public int getMaximumApplicationsPerQueue(String queue) {
+ int maxApplicationsPerQueue =
+ getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
+ (int)UNDEFINED);
+ return maxApplicationsPerQueue;
+ }
+
+ /**
+ * Get the maximum am resource percent per queue setting.
+ * @param queue name of the queue
+ * @return per queue setting or defaults to the global am-resource-percent
+ * setting if per queue setting not present
+ */
+ public float getMaximumApplicationMasterResourcePerQueuePercent(String
queue) {
+ return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
+ getMaximumApplicationMasterResourcePercent());
+ }
public float getCapacity(String queue) {
float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED);
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1364766&r1=1364765&r2=1364766&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Mon Jul 23 19:35:56 2012
@@ -85,7 +85,7 @@ public class LeafQueue implements CSQueu
private int maxApplications;
private int maxApplicationsPerUser;
- private float maxAMResourcePercent;
+ private float maxAMResourcePerQueuePercent;
private int maxActiveApplications; // Based on absolute max capacity
private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
@@ -155,21 +155,25 @@ public class LeafQueue implements CSQueu
float userLimitFactor =
cs.getConfiguration().getUserLimitFactor(getQueuePath());
- int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications();
- int maxApplications = (int)(maxSystemJobs * absoluteCapacity);
- int maxApplicationsPerUser =
+ int maxApplications =
cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
+ if (maxApplications < 0) {
+ int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
+ maxApplications = (int)(maxSystemApps * absoluteCapacity);
+ }
+ maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
- this.maxAMResourcePercent =
- cs.getConfiguration().getMaximumApplicationMasterResourcePercent();
+ this.maxAMResourcePerQueuePercent =
+ cs.getConfiguration().
+ getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
int maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
- maxAMResourcePercent, absoluteMaxCapacity);
+ maxAMResourcePerQueuePercent, absoluteMaxCapacity);
this.maxActiveAppsUsingAbsCap =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
- maxAMResourcePercent, absoluteCapacity);
+ maxAMResourcePerQueuePercent, absoluteCapacity);
int maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap,
userLimit,
userLimitFactor);
@@ -264,15 +268,16 @@ public class LeafQueue implements CSQueu
"userLimitFactor = " + userLimitFactor +
" [= configuredUserLimitFactor ]" + "\n" +
"maxApplications = " + maxApplications +
- " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]"
+
+ " [= configuredMaximumSystemApplicationsPerQueue or" +
+ " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" +
"\n" +
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
"maxActiveApplications = " + maxActiveApplications +
" [= max(" +
- "(int)ceil((clusterResourceMemory / minimumAllocation) *" +
- "maxAMResourcePercent * absoluteMaxCapacity)," +
+ "(int)ceil((clusterResourceMemory / minimumAllocation) * " +
+ "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," +
"1) ]" + "\n" +
"maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
" [= max(" +
@@ -289,7 +294,7 @@ public class LeafQueue implements CSQueu
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
" [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
- "maxAMResourcePercent = " + maxAMResourcePercent +
+ "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor +
" [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
@@ -1386,11 +1391,11 @@ public class LeafQueue implements CSQueu
maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
- maxAMResourcePercent, absoluteMaxCapacity);
+ maxAMResourcePerQueuePercent, absoluteMaxCapacity);
maxActiveAppsUsingAbsCap =
- CSQueueUtils.computeMaxActiveApplications(
- clusterResource, minimumAllocation,
- maxAMResourcePercent, absoluteCapacity);
+ CSQueueUtils.computeMaxActiveApplications(
+ clusterResource, minimumAllocation,
+ maxAMResourcePerQueuePercent, absoluteCapacity);
maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(
maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1364766&r1=1364765&r2=1364766&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
Mon Jul 23 19:35:56 2012
@@ -154,7 +154,9 @@ public class TestApplicationLimits {
int expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@@ -179,7 +181,9 @@ public class TestApplicationLimits {
expectedMaxActiveApps =
Math.max(1,
(int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
@@ -196,6 +200,72 @@ public class TestApplicationLimits {
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
+
+ // should return -1 if per queue setting not set
+ assertEquals((int)csConf.UNDEFINED,
csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ int expectedMaxApps = (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS *
+ queue.getAbsoluteCapacity());
+ assertEquals(expectedMaxApps, queue.getMaxApplications());
+
+ int expectedMaxAppsPerUser = (int)(expectedMaxApps *
+ (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+ assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
+
+ // should default to global setting if per queue setting not set
+ assertEquals((long)
csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ (long)
csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+
+ // Change the per-queue max AM resources percentage.
+ csConf.setFloat(
+ "yarn.scheduler.capacity." +
+ queue.getQueuePath() +
+ ".maximum-am-resource-percent",
+ 0.5f);
+ // Re-create queues to get new configs.
+ queues = new HashMap<String, CSQueue>();
+ root =
+ CapacityScheduler.parseQueue(csContext, csConf, null, "root",
+ queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ clusterResource = Resources.createResource(100 * 16 * GB);
+
+ queue = (LeafQueue)queues.get(A);
+ expectedMaxActiveApps =
+ Math.max(1,
+ (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
+ csConf.
+ getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()) *
+ queue.getAbsoluteMaximumCapacity()));
+
+ assertEquals((long) 0.5,
+ (long)
csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
+ assertEquals(expectedMaxActiveApps,
+ queue.getMaximumActiveApplications());
+
+ // Change the per-queue max applications.
+ csConf.setInt(
+ "yarn.scheduler.capacity." +
+ queue.getQueuePath() +
+ ".maximum-applications", 9999);
+ // Re-create queues to get new configs.
+ queues = new HashMap<String, CSQueue>();
+ root =
+ CapacityScheduler.parseQueue(csContext, csConf, null, "root",
+ queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+
+ queue = (LeafQueue)queues.get(A);
+ assertEquals(9999,
(int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ assertEquals(9999, queue.getMaxApplications());
+
+ expectedMaxAppsPerUser = (int)(9999 *
+ (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
+ assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
}
@Test
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1364766&r1=1364765&r2=1364766&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
Mon Jul 23 19:35:56 2012
@@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capac
*--------------------------------------+--------------------------------------+
|| Property || Description |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.maximum-applications>>> | |
+| <<<yarn.scheduler.capacity.maximum-applications>>> / |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>> | |
| | Maximum number of applications in the system which can be concurrently |
| | active both running and pending. Limits on each queue are directly |
| | proportional to their queue capacities and user limits. This is a
| | hard limit and any applications submitted when this limit is reached will |
-| | be rejected. Default is 10000.|
+| | be rejected. Default is 10000. This can be set for all queues with |
+| | <<<yarn.scheduler.capacity.maximum-applications>>> and can also be
overridden on a |
+| | per queue basis by setting
<<<yarn.scheduler.capacity.<queue-path>.maximum-applications>>>. |
*--------------------------------------+--------------------------------------+
-| yarn.scheduler.capacity.maximum-am-resource-percent | |
+| <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> / |
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> | |
| | Maximum percent of resources in the cluster which can be used to run |
-| | application masters - controls number of concurrent running applications. |
-| | Specified as a float - ie 0.5 = 50%. Default is 10%. |
+| | application masters - controls number of concurrent active applications.
Limits on each |
+| | queue are directly proportional to their queue capacities and user limits.
|
+| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for
all queues with |
+| | <<<yarn.scheduler.capacity.maximum-am-resource-percent>>> and can also be
overridden on a |
+| | per queue basis by setting
<<<yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent>>> |
*--------------------------------------+--------------------------------------+
* Queue Administration & Permissions
