brumi1024 commented on code in PR #8802: URL: https://github.com/apache/hadoop/pull/8802#discussion_r4263700819
########## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityChecks.java: ########## @@ -0,0 +1,491 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue.QueueCapacityType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.ResourceUnitCapacityType; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Capacity checks run while a queue is built from configuration (legacy queue + * mode type and sum checks, and configured min/max resource checks). + * Every method returns null when the check passes, or the exact message of the + * exception the caller throws when it fails, so a validator working on resolved + * configuration can share the same implementation as the queue constructors. + */ [email protected] [email protected] +public final class QueueCapacityChecks { + /** + * Tolerance used when comparing the sum of children capacities with 0 or 1. + */ + public static final float CAPACITY_SUM_PRECISION = 0.0005f; + + private QueueCapacityChecks() {} + + /** + * The capacity configuration type of one label in legacy queue mode. + * @param absoluteResource whether the capacity of the label is configured + * as an absolute resource + * @return ABSOLUTE_RESOURCE or PERCENTAGE + */ + public static CapacityConfigType capacityConfigTypeOf( + boolean absoluteResource) { + return absoluteResource ? CapacityConfigType.ABSOLUTE_RESOURCE + : CapacityConfigType.PERCENTAGE; + } + + /** + * The capacity configuration type of one label outside legacy queue mode: + * ABSOLUTE_RESOURCE when every resource is absolute, otherwise PERCENTAGE + * (percentage, weight and mixed vectors). + * @param capacityVector the configured capacity vector of the label + * @return ABSOLUTE_RESOURCE or PERCENTAGE + */ + public static CapacityConfigType capacityConfigTypeOf( + QueueCapacityVector capacityVector) { + Set<ResourceUnitCapacityType> definedCapacityTypes = + capacityVector.getDefinedCapacityTypes(); + if (definedCapacityTypes.size() == 1 && definedCapacityTypes.iterator() + .next() == ResourceUnitCapacityType.ABSOLUTE) { + return CapacityConfigType.ABSOLUTE_RESOURCE; + } + return CapacityConfigType.PERCENTAGE; + } + + /** + * The weight a capacity vector defines when it only has weights and every + * resource has the same one, for example 5w == [memory=5w, vcores=5w]. + * Queue setup stores this weight in place of the configured weight. + * @param capacityVector the configured capacity vector of the label + * @return the common weight, or null if the vector does not define one + */ + public static Float uniformWeightOf(QueueCapacityVector capacityVector) { + Set<ResourceUnitCapacityType> definedCapacityTypes = + capacityVector.getDefinedCapacityTypes(); + if (definedCapacityTypes.size() != 1 || definedCapacityTypes.iterator() + .next() != ResourceUnitCapacityType.WEIGHT) { + return null; + } + Set<Double> weights = new HashSet<>(); + for (String resourceName : capacityVector.getResourceNames()) { + weights.add(capacityVector.getResource(resourceName).getResourceValue()); + } + return weights.size() == 1 ? weights.iterator().next().floatValue() : null; + } + + /** + * Capacity related values of one queue for one node label, as seen by the + * legacy queue mode capacity type checks. + */ + public static final class LabelCapacity { + /** Values reported by QueueCapacities for a label it does not know. */ + public static final LabelCapacity UNSET = new LabelCapacity(0f, -1f, false); + + private final float capacity; + private final float weight; + private final boolean absolute; + + /** + * @param capacity relative capacity in the range [0, 1] + * @param weight configured weight, -1 when no weight is configured + * @param absolute whether the capacity of the label is configured as an + * absolute resource + */ + public LabelCapacity(float capacity, float weight, boolean absolute) { + this.capacity = capacity; + this.weight = weight; + this.absolute = absolute; + } + + public float getCapacity() { + return capacity; + } + + public float getWeight() { + return weight; + } + + public boolean isAbsolute() { + return absolute; + } + } + + /** + * Per label capacity values of one queue. + */ + public static final class QueueCapacityInput { + private final String queuePath; + private final Map<String, LabelCapacity> byLabel; + + public QueueCapacityInput(String queuePath, + Map<String, LabelCapacity> byLabel) { + this.queuePath = queuePath; + this.byLabel = Collections.unmodifiableMap(new HashMap<>(byLabel)); + } + + public String getQueuePath() { + return queuePath; + } + + /** + * @param label node label + * @return the values of the label, or {@link LabelCapacity#UNSET} + */ + public LabelCapacity get(String label) { + LabelCapacity labelCapacity = byLabel.get(label); + return labelCapacity == null ? LabelCapacity.UNSET : labelCapacity; + } + } + + /** + * C05: the raw capacity of the root queue must be 100. Review Comment: Good catch, those came from my local inventory. Removed them as they are not relevant for the PR. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
