YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan R.
(cherry picked from commit 805095496dfd8e50f71b70cf20845e954d3ba47c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cc75e90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cc75e90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cc75e90 Branch: refs/heads/YARN-8200 Commit: 6cc75e901e836312bd994e264160a8cdcb4d55e8 Parents: edf7ea6 Author: Sunil G <sun...@apache.org> Authored: Wed Sep 6 18:51:14 2017 +0530 Committer: Konstantin V Shvachko <kshvac...@linkedin.com> Committed: Wed May 2 13:19:58 2018 -0700 ---------------------------------------------------------------------- .../yarn/util/resource/ResourceUtils.java | 38 ++++++++++---------- .../resource-types/resource-types-error-2.xml | 4 +++ .../resource-types/resource-types-error-3.xml | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc75e90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 94c2e97..e3e25d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -42,10 +42,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** @@ -63,13 +61,6 @@ public class ResourceUtils { private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); private static final String VCORES = ResourceInformation.VCORES.getName(); - private static final Set<String> DISALLOWED_NAMES = new HashSet<>(); - static { - DISALLOWED_NAMES.add("memory"); - DISALLOWED_NAMES.add(MEMORY); - DISALLOWED_NAMES.add(VCORES); - } - private static volatile boolean initializedResources = false; private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX = new ConcurrentHashMap<String, Integer>(); @@ -85,9 +76,21 @@ public class ResourceUtils { private ResourceUtils() { } - private static void checkMandatatoryResources( + private static void checkMandatoryResources( Map<String, ResourceInformation> resourceInformationMap) throws YarnRuntimeException { + /* + * Supporting 'memory' also as invalid resource name, in addition to + * 'MEMORY' for historical reasons + */ + String key = "memory"; + if (resourceInformationMap.containsKey(key)) { + LOG.warn("Attempt to define resource '" + key + + "', but it is not allowed."); + throw new YarnRuntimeException("Attempt to re-define mandatory resource '" + + key + "'."); + } + if (resourceInformationMap.containsKey(MEMORY)) { ResourceInformation memInfo = resourceInformationMap.get(MEMORY); String memUnits = ResourceInformation.MEMORY_MB.getUnits(); @@ -113,7 +116,7 @@ public class ResourceUtils { } } - private static void addManadtoryResources( + private static void addMandatoryResources( Map<String, ResourceInformation> res) { ResourceInformation ri; if (!res.containsKey(MEMORY)) { @@ -229,11 +232,6 @@ public class ResourceUtils { "Incomplete configuration for resource type '" + resourceName + "'. One of name, units or type is configured incorrectly."); } - if (DISALLOWED_NAMES.contains(resourceName)) { - throw new YarnRuntimeException( - "Resource type cannot be named '" + resourceName - + "'. That name is disallowed."); - } ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); LOG.info("Adding resource type - name = " + resourceName + ", units = " + resourceUnits + ", type = " + resourceTypeName); @@ -246,8 +244,8 @@ public class ResourceUtils { minimumAllocation, maximumAllocation)); } } - checkMandatatoryResources(resourceInformationMap); - addManadtoryResources(resourceInformationMap); + checkMandatoryResources(resourceInformationMap); + addMandatoryResources(resourceInformationMap); setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); setMaximumAllocationForMandatoryResources(resourceInformationMap, conf); resourceTypes = Collections.unmodifiableMap(resourceInformationMap); @@ -454,8 +452,8 @@ public class ResourceUtils { if (!initializedNodeResources) { Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation( conf); - addManadtoryResources(nodeResources); - checkMandatatoryResources(nodeResources); + addMandatoryResources(nodeResources); + checkMandatoryResources(nodeResources); setMinimumAllocationForMandatoryResources(nodeResources, conf); setMaximumAllocationForMandatoryResources(nodeResources, conf); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc75e90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml index ca428eb..fa43b6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml @@ -26,4 +26,8 @@ limitations under the License. See accompanying LICENSE file. <value>G</value> </property> + <property> + <name>yarn.resource-types.vcores.units</name> + <value>Az</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc75e90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml index 08b8a6d..539d657 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml @@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file. <property> <name>yarn.resource-types</name> - <value>vcores,resource1</value> + <value>resource1,resource1</value> </property> <property> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org