YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b286228 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b286228 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b286228 Branch: refs/heads/HDFS-10467 Commit: 8b2862283767b031d1e17e3db812fb9cdad552ad Parents: 7d33e51 Author: Wangda Tan <wan...@apache.org> Authored: Tue Apr 11 14:56:18 2017 -0700 Committer: Inigo <inigo...@apache.org> Committed: Mon Apr 17 11:17:01 2017 -0700 ---------------------------------------------------------------------- .../capacity/CapacitySchedulerQueueManager.java | 20 ++++++++++++++++++++ .../scheduler/capacity/PlanQueue.java | 4 ++-- .../TestCapacitySchedulerPlanFollower.java | 7 +++++++ .../TestFairSchedulerPlanFollower.java | 4 ++++ .../TestSchedulerPlanFollowerBase.java | 9 ++++++++- .../TestCapacitySchedulerDynamicBehavior.java | 11 +++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index c92c343..be6243d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -41,10 +41,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.Permission; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import com.google.common.annotations.VisibleForTesting; @@ -220,6 +223,23 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager< queue = new PlanQueue(csContext, queueName, parent, oldQueues.get(queueName)); + + //initializing the "internal" default queue, for SLS compatibility + String defReservationId = + queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; + + List<CSQueue> childQueues = new ArrayList<>(); + ReservationQueue resQueue = new ReservationQueue(csContext, + defReservationId, (PlanQueue) queue); + try { + resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f)); + } catch (SchedulerDynamicEditException e) { + throw new IllegalStateException(e); + } + childQueues.add(resQueue); + ((PlanQueue) queue).setChildQueues(childQueues); + queues.put(defReservationId, resQueue); + } else { queue = new LeafQueue(csContext, queueName, parent, http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index a391f25..882262f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -93,10 +93,10 @@ public class PlanQueue extends ParentQueue { PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue; - if (newlyParsedParentQueue.getChildQueues().size() > 0) { + if (newlyParsedParentQueue.getChildQueues().size() != 1) { throw new IOException( "Reservable Queue should not have sub-queues in the" - + "configuration"); + + "configuration expect the default reservation queue"); } // Set new configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index bccd5c3..4320d3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -154,6 +154,13 @@ public class TestCapacitySchedulerPlanFollower extends } @Override + protected void checkDefaultQueueBeforePlanFollowerRun(){ + Queue defQ = getDefaultQueue(); + Assert.assertEquals(0, getNumberOfApplications(defQ)); + Assert.assertNotNull(defQ); + } + + @Override protected Queue getDefaultQueue() { return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java index 1b4e2f8..9561234 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java @@ -132,6 +132,10 @@ public class TestFairSchedulerPlanFollower extends } @Override + protected void checkDefaultQueueBeforePlanFollowerRun() { + Assert.assertNull(getDefaultQueue()); + } + @Override protected void verifyCapacity(Queue defQ) { assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java index b604799..c6cebd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java @@ -89,6 +89,11 @@ public abstract class TestSchedulerPlanFollowerBase { "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil .generateAllocation(10L, 1L, f2), res, minAlloc), false)); + + // default reseration queue should exist before run of PlanFollower AND have + // no apps + checkDefaultQueueBeforePlanFollowerRun(); + AbstractSchedulerPlanFollower planFollower = createPlanFollower(); when(mClock.getTime()).thenReturn(0L); @@ -108,8 +113,8 @@ public abstract class TestSchedulerPlanFollowerBase { new AppAttemptAddedSchedulerEvent(appAttemptId_0, false); scheduler.handle(appAttemptAddedEvent); - // initial default reservation queue should have no apps + // initial default reservation queue should have no apps after first run Queue defQ = getDefaultQueue(); Assert.assertEquals(0, getNumberOfApplications(defQ)); @@ -179,6 +184,8 @@ public abstract class TestSchedulerPlanFollowerBase { verifyCapacity(defQ); } + protected abstract void checkDefaultQueueBeforePlanFollowerRun(); + protected abstract Queue getReservationQueue(String reservationId); protected abstract void verifyCapacity(Queue defQ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index ce3382f..483ba1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -76,6 +76,12 @@ public class TestCapacitySchedulerDynamicBehavior { public void testRefreshQueuesWithReservations() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); + // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); @@ -120,6 +126,11 @@ public class TestCapacitySchedulerDynamicBehavior { ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
