This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new aa74a30 YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420) aa74a30 is described below commit aa74a303ed30a057893c6d2d9fb1e07e7d1f4a7d Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com> AuthorDate: Sun Sep 19 13:11:56 2021 +0200 YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420) --- .../scheduler/capacity/AbstractCSQueue.java | 100 ++--------------- .../scheduler/capacity/CSQueuePreemption.java | 119 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 90 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 75aebfe..f0d0639 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -106,10 +105,6 @@ public abstract class AbstractCSQueue implements CSQueue { Map<AccessType, AccessControlList> acls = new HashMap<AccessType, AccessControlList>(); volatile boolean reservationsContinueLooking; - private volatile boolean preemptionDisabled; - // Indicates if the in-queue preemption setting is ever disabled within the - // hierarchy of this queue. - private boolean intraQueuePreemptionDisabledInHierarchy; // Track resource usage-by-label like used-resource/pending-resource, etc. volatile ResourceUsage queueUsage; @@ -129,6 +124,7 @@ public abstract class AbstractCSQueue implements CSQueue { // Indicates if this queue's default lifetime was set by a config property, // either at this level or anywhere in the queue's hierarchy. private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; + private CSQueuePreemption preemptionSettings; public enum CapacityConfigType { // FIXME, from what I can see, Percentage mode can almost apply to weighted @@ -401,10 +397,7 @@ public abstract class AbstractCSQueue implements CSQueue { this, labelManager, null); // Store preemption settings - this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this, - configuration); - this.intraQueuePreemptionDisabledInHierarchy = - isIntraQueueHierarchyPreemptionDisabled(this, configuration); + this.preemptionSettings = new CSQueuePreemption(this, csContext, configuration); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -777,7 +770,7 @@ public abstract class AbstractCSQueue implements CSQueue { queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression); queueInfo.setCurrentCapacity(getUsedCapacity()); queueInfo.setQueueStatistics(getQueueStatistics()); - queueInfo.setPreemptionDisabled(preemptionDisabled); + queueInfo.setPreemptionDisabled(getPreemptionDisabled()); queueInfo.setIntraQueuePreemptionDisabled( getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); @@ -902,17 +895,18 @@ public abstract class AbstractCSQueue implements CSQueue { @Private public boolean getPreemptionDisabled() { - return preemptionDisabled; + return preemptionSettings.isPreemptionDisabled(); } @Private public boolean getIntraQueuePreemptionDisabled() { - return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled; + return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() || + preemptionSettings.isPreemptionDisabled(); } @Private public boolean getIntraQueuePreemptionDisabledInHierarchy() { - return intraQueuePreemptionDisabledInHierarchy; + return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy(); } @Private @@ -935,43 +929,6 @@ public abstract class AbstractCSQueue implements CSQueue { return readLock; } - /** - * The specified queue is cross-queue preemptable if system-wide cross-queue - * preemption is turned on unless any queue in the <em>qPath</em> hierarchy - * has explicitly turned cross-queue preemption off. - * NOTE: Cross-queue preemptability is inherited from a queue's parent. - * - * @param q queue to check preemption state - * @param configuration capacity scheduler config - * @return true if queue has cross-queue preemption disabled, false otherwise - */ - private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerConfiguration configuration) { - boolean systemWidePreemption = - csContext.getConfiguration() - .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); - CSQueue parentQ = q.getParent(); - - // If the system-wide preemption switch is turned off, all of the queues in - // the qPath hierarchy have preemption disabled, so return true. - if (!systemWidePreemption) return true; - - // If q is the root queue and the system-wide preemption switch is turned - // on, then q does not have preemption disabled (default=false, below) - // unless the preemption_disabled property is explicitly set. - if (parentQ == null) { - return configuration.getPreemptionDisabled(q.getQueuePath(), false); - } - - // If this is not the root queue, inherit the default value for the - // preemption_disabled property from the parent. Preemptability will be - // inherited from the parent's hierarchy unless explicitly overridden at - // this level. - return configuration.getPreemptionDisabled(q.getQueuePath(), - parentQ.getPreemptionDisabled()); - } - private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) { CSQueue parentQ = q.getParent(); @@ -1042,43 +999,6 @@ public abstract class AbstractCSQueue implements CSQueue { return defaultAppLifetime; } - /** - * The specified queue is intra-queue preemptable if - * 1) system-wide intra-queue preemption is turned on - * 2) no queue in the <em>qPath</em> hierarchy has explicitly turned off intra - * queue preemption. - * NOTE: Intra-queue preemptability is inherited from a queue's parent. - * - * @param q queue to check intra-queue preemption state - * @param configuration capacity scheduler config - * @return true if queue has intra-queue preemption disabled, false otherwise - */ - private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, - CapacitySchedulerConfiguration configuration) { - boolean systemWideIntraQueuePreemption = - csContext.getConfiguration().getBoolean( - CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, - CapacitySchedulerConfiguration - .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); - // Intra-queue preemption is disabled for this queue if the system-wide - // intra-queue preemption flag is false - if (!systemWideIntraQueuePreemption) return true; - - // Check if this is the root queue and the root queue's intra-queue - // preemption disable switch is set - CSQueue parentQ = q.getParent(); - if (parentQ == null) { - return configuration - .getIntraQueuePreemptionDisabled(q.getQueuePath(), false); - } - - // At this point, the master preemption switch is enabled down to this - // queue's level. Determine whether or not intra-queue preemption is enabled - // down to this queue's level and return that value. - return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(), - parentQ.getIntraQueuePreemptionDisabledInHierarchy()); - } - private Resource getCurrentLimitResource(String nodePartition, Resource clusterResource, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { @@ -1423,11 +1343,11 @@ public abstract class AbstractCSQueue implements CSQueue { LOG.info("The specified queue:" + getQueuePath() + " is already in the RUNNING state."); } else { - CSQueue parent = getParent(); - if (parent == null || parent.getState() == QueueState.RUNNING) { + CSQueue parentQueue = getParent(); + if (parentQueue == null || parentQueue.getState() == QueueState.RUNNING) { updateQueueState(QueueState.RUNNING); } else { - throw new YarnException("The parent Queue:" + parent.getQueuePath() + throw new YarnException("The parent Queue:" + parentQueue.getQueuePath() + " is not running. Please activate the parent queue first"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java new file mode 100644 index 0000000..0a11bcd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class CSQueuePreemption { + private final boolean preemptionDisabled; + // Indicates if the in-queue preemption setting is ever disabled within the + // hierarchy of this queue. + private final boolean intraQueuePreemptionDisabledInHierarchy; + + public CSQueuePreemption( + CSQueue queue, + CapacitySchedulerContext csContext, + CapacitySchedulerConfiguration configuration) { + this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, csContext, configuration); + this.intraQueuePreemptionDisabledInHierarchy = + isIntraQueueHierarchyPreemptionDisabled(queue, csContext, configuration); + } + + /** + * The specified queue is cross-queue preemptable if system-wide cross-queue + * preemption is turned on unless any queue in the <em>qPath</em> hierarchy + * has explicitly turned cross-queue preemption off. + * NOTE: Cross-queue preemptability is inherited from a queue's parent. + * + * @param q queue to check preemption state + * @param csContext + * @param configuration capacity scheduler config + * @return true if queue has cross-queue preemption disabled, false otherwise + */ + private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, + CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { + boolean systemWidePreemption = + csContext.getConfiguration() + .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); + CSQueue parentQ = q.getParent(); + + // If the system-wide preemption switch is turned off, all of the queues in + // the qPath hierarchy have preemption disabled, so return true. + if (!systemWidePreemption) return true; + + // If q is the root queue and the system-wide preemption switch is turned + // on, then q does not have preemption disabled (default=false, below) + // unless the preemption_disabled property is explicitly set. + if (parentQ == null) { + return configuration.getPreemptionDisabled(q.getQueuePath(), false); + } + + // If this is not the root queue, inherit the default value for the + // preemption_disabled property from the parent. Preemptability will be + // inherited from the parent's hierarchy unless explicitly overridden at + // this level. + return configuration.getPreemptionDisabled(q.getQueuePath(), + parentQ.getPreemptionDisabled()); + } + + /** + * The specified queue is intra-queue preemptable if + * 1) system-wide intra-queue preemption is turned on + * 2) no queue in the <em>qPath</em> hierarchy has explicitly turned off intra + * queue preemption. + * NOTE: Intra-queue preemptability is inherited from a queue's parent. + * + * @param q queue to check intra-queue preemption state + * @param csContext + * @param configuration capacity scheduler config + * @return true if queue has intra-queue preemption disabled, false otherwise + */ + private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, + CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { + boolean systemWideIntraQueuePreemption = + csContext.getConfiguration().getBoolean( + CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, + CapacitySchedulerConfiguration + .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); + // Intra-queue preemption is disabled for this queue if the system-wide + // intra-queue preemption flag is false + if (!systemWideIntraQueuePreemption) return true; + + // Check if this is the root queue and the root queue's intra-queue + // preemption disable switch is set + CSQueue parentQ = q.getParent(); + if (parentQ == null) { + return configuration + .getIntraQueuePreemptionDisabled(q.getQueuePath(), false); + } + + // At this point, the master preemption switch is enabled down to this + // queue's level. Determine whether intra-queue preemption is enabled + // down to this queue's level and return that value. + return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(), + parentQ.getIntraQueuePreemptionDisabledInHierarchy()); + } + + public boolean isIntraQueuePreemptionDisabledInHierarchy() { + return intraQueuePreemptionDisabledInHierarchy; + } + + public boolean isPreemptionDisabled() { + return preemptionDisabled; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org