This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new cfe6e1f YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori cfe6e1f is described below commit cfe6e1f7da95d2ae05e55e5244925d296df4bbd4 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Wed Jan 20 15:22:44 2021 +0100 YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java | 19 ---------------- .../scheduler/capacity/CapacityScheduler.java | 25 +++++++++++----------- .../CapacitySchedulerAutoQueueHandler.java | 5 +---- .../TestCapacitySchedulerNewQueueAutoCreation.java | 3 +-- 4 files changed, 14 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index c02e1d3..bc3ff22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -154,11 +154,6 @@ public abstract class AbstractCSQueue implements CSQueue { // is it a dynamic queue? private boolean dynamicQueue = false; - // When this queue has application submit to? - // This property only applies to dynamic queue, - // and will be used to check when the queue need to be removed. - private long lastSubmittedTimestamp; - public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); @@ -1633,18 +1628,4 @@ public abstract class AbstractCSQueue implements CSQueue { writeLock.unlock(); } } - - public long getLastSubmittedTimestamp() { - return lastSubmittedTimestamp; - } - - // "Tab" the queue, so this queue won't be removed because of idle timeout. - public void signalToSubmitToQueue() { - writeLock.lock(); - try { - this.lastSubmittedTimestamp = System.currentTimeMillis(); - } finally { - writeLock.unlock(); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 5ee557a..204fa73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -343,7 +343,7 @@ public class CapacityScheduler extends this.queueManager.setCapacitySchedulerContext(this); this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler( - this.queueManager, this.conf); + this.queueManager); this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager(); @@ -3380,26 +3380,25 @@ public class CapacityScheduler extends if (!StringUtils.isEmpty(parentQueueName)) { CSQueue parentQueue = getQueue(parentQueueName); - if (parentQueue == null) { - throw new SchedulerDynamicEditException( - "Could not auto-create leaf queue for " + leafQueueName - + ". Queue mapping specifies an invalid parent queue " - + "which does not exist " + parentQueueName); - } - - if (conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) { + if (parentQueue != null && + conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) { // Case 1: Handle ManagedParentQueue - AutoCreatedLeafQueue autoCreatedLeafQueue = null; ManagedParentQueue autoCreateEnabledParentQueue = (ManagedParentQueue) parentQueue; - autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName, - autoCreateEnabledParentQueue); + AutoCreatedLeafQueue autoCreatedLeafQueue = + new AutoCreatedLeafQueue( + this, leafQueueName, autoCreateEnabledParentQueue); addQueue(autoCreatedLeafQueue); return autoCreatedLeafQueue; } else { - return autoQueueHandler.autoCreateQueue(placementContext); + try { + writeLock.lock(); + return autoQueueHandler.autoCreateQueue(placementContext); + } finally { + writeLock.unlock(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java index 1730021..55ab1ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java @@ -31,14 +31,11 @@ import java.util.List; */ public class CapacitySchedulerAutoQueueHandler { private final CapacitySchedulerQueueManager queueManager; - private final CapacitySchedulerConfiguration conf; private static final int MAXIMUM_DEPTH_ALLOWED = 2; public CapacitySchedulerAutoQueueHandler( - CapacitySchedulerQueueManager queueManager, - CapacitySchedulerConfiguration conf) { + CapacitySchedulerQueueManager queueManager) { this.queueManager = queueManager; - this.conf = conf; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 25b2f4d..98b6d3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -83,7 +83,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation mockRM.start(); cs.start(); autoQueueHandler = new CapacitySchedulerAutoQueueHandler( - cs.getCapacitySchedulerQueueManager(), csConf); + cs.getCapacitySchedulerQueueManager()); mockRM.registerNode("h1:1234", MAX_MEMORY * GB); // label = x } @@ -409,7 +409,6 @@ public class TestCapacitySchedulerNewQueueAutoCreation @Test public void testAutoQueueCreationOnAppSubmission() throws Exception { startScheduler(); - createBasicQueueStructureAndValidate(); submitApp(cs, USER0, USER0, "root.e-auto"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org