This is an automated email from the ASF dual-hosted git repository. pbacsko pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 7e8040e YARN-10655. Limit queue creation depth relative to its first static parent. Contributed by Andras Gyori. 7e8040e is described below commit 7e8040e6adccad55a603c1ecd513a02e7edc7425 Author: Peter Bacsko <pbac...@cloudera.com> AuthorDate: Wed Mar 3 17:44:30 2021 +0100 YARN-10655. Limit queue creation depth relative to its first static parent. Contributed by Andras Gyori. --- .../CapacitySchedulerAutoQueueHandler.java | 40 +++++++++++++++------- .../TestCapacitySchedulerNewQueueAutoCreation.java | 20 +++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java index e847737..898b075 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java @@ -55,34 +55,43 @@ public class CapacitySchedulerAutoQueueHandler { List<ApplicationPlacementContext> parentsToCreate = new ArrayList<>(); ApplicationPlacementContext queueCandidateContext = parentContext; - CSQueue existingQueueCandidate = getQueue( + CSQueue firstExistingQueue = getQueue( queueCandidateContext.getFullQueuePath()); - while (existingQueueCandidate == null) { + while (firstExistingQueue == null) { parentsToCreate.add(queueCandidateContext); queueCandidateContext = CSQueueUtils.extractQueuePath( queueCandidateContext.getParentQueue()); - existingQueueCandidate = getQueue( + firstExistingQueue = getQueue( queueCandidateContext.getFullQueuePath()); } + CSQueue firstExistingStaticQueue = firstExistingQueue; + // Include the LeafQueue in the distance + int firstStaticParentDistance = parentsToCreate.size() + 1; + + while(isNonStaticParent(firstExistingStaticQueue)) { + queueCandidateContext = CSQueueUtils.extractQueuePath( + queueCandidateContext.getParentQueue()); + firstExistingStaticQueue = getQueue( + queueCandidateContext.getFullQueuePath()); + ++firstStaticParentDistance; + } + // Reverse the collection to to represent the hierarchy to be created // from highest to lowest level Collections.reverse(parentsToCreate); - if (!(existingQueueCandidate instanceof ParentQueue)) { + if (!(firstExistingQueue instanceof ParentQueue)) { throw new SchedulerDynamicEditException( "Could not auto create hierarchy of " + queue.getFullQueuePath() + ". Queue " - + existingQueueCandidate.getQueuePath() + + + firstExistingQueue.getQueuePath() + " is not a ParentQueue." ); } - ParentQueue existingParentQueue = (ParentQueue) existingQueueCandidate; + ParentQueue existingParentQueue = (ParentQueue) firstExistingQueue; int depthLimit = extractDepthLimit(existingParentQueue); - // The number of levels to be created including the LeafQueue - // (which is last) - int levelsToCreate = parentsToCreate.size() + 1; if (depthLimit == 0) { throw new SchedulerDynamicEditException("Auto creation of queue " + @@ -90,12 +99,12 @@ public class CapacitySchedulerAutoQueueHandler { + existingParentQueue.getQueuePath()); } - if (levelsToCreate > depthLimit) { + if (firstStaticParentDistance > depthLimit) { throw new SchedulerDynamicEditException( "Could not auto create queue " + queue.getFullQueuePath() - + ". In order to create the desired queue hierarchy, " + - levelsToCreate + " levels of queues would need " + - "to be created, which is above the limit."); + + ". The distance of the LeafQueue from the first static " + + "ParentQueue is" + firstStaticParentDistance + ", which is " + + "above the limit."); } for (ApplicationPlacementContext current : parentsToCreate) { @@ -123,4 +132,9 @@ public class CapacitySchedulerAutoQueueHandler { private CSQueue getQueue(String queue) { return queue != null ? queueManager.getQueue(queue) : null; } + + private boolean isNonStaticParent(CSQueue queue) { + return (!(queue instanceof AbstractCSQueue) + || ((AbstractCSQueue) queue).isDynamicQueue()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 4facf94..2f83f1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -565,6 +565,26 @@ public class TestCapacitySchedulerNewQueueAutoCreation } } + @Test + public void testAutoQueueCreationDepthLimitFromStaticParent() + throws Exception { + startScheduler(); + + // a is the first existing queue here and it is static, therefore + // the distance is 2 + createQueue("root.a.a-auto.a1-auto"); + Assert.assertNotNull(cs.getQueue("root.a.a-auto.a1-auto")); + + try { + createQueue("root.a.a-auto.a2-auto.a3-auto"); + Assert.fail("Queue creation should not succeed because the distance " + + "from the first static parent is above limit"); + } catch (SchedulerDynamicEditException ignored) { + + } + + } + private LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath)); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org