Author: bobby Date: Wed Oct 31 17:03:45 2012 New Revision: 1404251 URL: http://svn.apache.org/viewvc?rev=1404251&view=rev Log: svn merge -c 1404247 FIXES: YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via bobby)
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Wed Oct 31 17:03:45 2012 @@ -62,6 +62,9 @@ Release 0.23.5 - UNRELEASED YARN-159. RM web ui applications page should be sorted to display last app first (tgraves via bobby) + YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via + bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Wed Oct 31 17:03:45 2012 @@ -84,7 +84,7 @@ class CSQueueUtils { if (clusterMemory > 0) { queueLimit = clusterMemory * childQueue.getAbsoluteCapacity(); absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory); - usedCapacity = (usedMemory / queueLimit); + usedCapacity = (queueLimit == 0) ? 0 : (usedMemory / queueLimit); } childQueue.setUsedCapacity(usedCapacity); Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Wed Oct 31 17:03:45 2012 @@ -91,7 +91,7 @@ public class CapacitySchedulerConfigurat public static final float UNDEFINED = -1; @Private - public static final float MINIMUM_CAPACITY_VALUE = 1; + public static final float MINIMUM_CAPACITY_VALUE = 0; @Private public static final float MAXIMUM_CAPACITY_VALUE = 100; Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Wed Oct 31 17:03:45 2012 @@ -202,7 +202,9 @@ public class ParentQueue implements CSQu childCapacities += queue.getCapacity(); } float delta = Math.abs(1.0f - childCapacities); // crude way to check - if (delta > PRECISION) { + // allow capacities being set to 0, and enforce child 0 if parent is 0 + if (((capacity > 0) && (delta > PRECISION)) || + ((capacity == 0) && (childCapacities > 0))) { throw new IllegalArgumentException("Illegal" + " capacity of " + childCapacities + " for children of queue " + queueName); Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Wed Oct 31 17:03:45 2012 @@ -65,8 +65,8 @@ public class TestCapacityScheduler { private static float B_CAPACITY = 89.5f; private static float A1_CAPACITY = 30; private static float A2_CAPACITY = 70; - private static float B1_CAPACITY = 50; - private static float B2_CAPACITY = 30; + private static float B1_CAPACITY = 79.2f; + private static float B2_CAPACITY = 0.8f; private static float B3_CAPACITY = 20; private ResourceManager resourceManager = null; Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Wed Oct 31 17:03:45 2012 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; @@ -496,6 +497,72 @@ public class TestParentQueue { verifyQueueMetrics(c, 4*GB, clusterResource); reset(a); reset(b); reset(c); } + + @Test (expected=IllegalArgumentException.class) + public void testQueueCapacitySettingChildZero() throws Exception { + // Setup queue configs + setupMultiLevelQueues(csConf); + + // set child queues capacity to 0 when parents not 0 + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; + csConf.setCapacity(Q_B + "." + B1, 0); + csConf.setCapacity(Q_B + "." + B2, 0); + csConf.setCapacity(Q_B + "." + B3, 0); + + Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + } + + @Test (expected=IllegalArgumentException.class) + public void testQueueCapacitySettingParentZero() throws Exception { + // Setup queue configs + setupMultiLevelQueues(csConf); + + // set parent capacity to 0 when child not 0 + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; + csConf.setCapacity(Q_B, 0); + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; + csConf.setCapacity(Q_A, 60); + + Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + } + + @Test + public void testQueueCapacityZero() throws Exception { + // Setup queue configs + setupMultiLevelQueues(csConf); + + // set parent and child capacity to 0 + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; + csConf.setCapacity(Q_B, 0); + csConf.setCapacity(Q_B + "." + B1, 0); + csConf.setCapacity(Q_B + "." + B2, 0); + csConf.setCapacity(Q_B + "." + B3, 0); + + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; + csConf.setCapacity(Q_A, 60); + + Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); + try { + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + } catch (IllegalArgumentException e) { + fail("Failed to create queues with 0 capacity: " + e); + } + assertTrue("Failed to create queues with 0 capacity", true); + } @Test public void testOffSwitchScheduling() throws Exception { Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java?rev=1404251&r1=1404250&r2=1404251&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java Wed Oct 31 17:03:45 2012 @@ -145,11 +145,11 @@ public class TestRMWebServicesCapacitySc final String B2 = B + ".b2"; final String B3 = B + ".b3"; conf.setQueues(B, new String[] { "b1", "b2", "b3" }); - conf.setCapacity(B1, 50); + conf.setCapacity(B1, 60); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2, 30); + conf.setCapacity(B2, 39.5f); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, 20); + conf.setCapacity(B3, 0.5f); conf.setUserLimitFactor(B3, 100.0f); conf.setQueues(A1, new String[] {"a1a", "a1b"});