YARN-6953. Clean up ResourceUtils.setMinimumAllocationForMandatoryResources() and setMaximumAllocationForMandatoryResources() (Contributed by Manikandan R via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e094eb74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e094eb74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e094eb74 Branch: refs/heads/YARN-5881 Commit: e094eb74b9e7d8c3c6f1990445d248b062cc230b Parents: 69043ba Author: Daniel Templeton <templ...@apache.org> Authored: Wed Nov 15 09:55:40 2017 -0800 Committer: Daniel Templeton <templ...@apache.org> Committed: Wed Nov 15 10:03:29 2017 -0800 ---------------------------------------------------------------------- .../yarn/util/resource/ResourceUtils.java | 108 +++++++------------ 1 file changed, 38 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e094eb74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 3deace8..c168337 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -142,74 +142,44 @@ public class ResourceUtils { } } - private static void setMinimumAllocationForMandatoryResources( + private static void setAllocationForMandatoryResources( Map<String, ResourceInformation> res, Configuration conf) { - String[][] resourceTypesKeys = { - {ResourceInformation.MEMORY_MB.getName(), - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), - ResourceInformation.MEMORY_MB.getName()}, - {ResourceInformation.VCORES.getName(), - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES), - ResourceInformation.VCORES.getName()}}; - for (String[] arr : resourceTypesKeys) { - String resourceTypesKey = - YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION; - long minimumResourceTypes = conf.getLong(resourceTypesKey, -1); - long minimumConf = conf.getLong(arr[1], -1); - long minimum; - if (minimumResourceTypes != -1) { - minimum = minimumResourceTypes; - if (minimumConf != -1) { - LOG.warn("Using minimum allocation for memory specified in " - + "resource-types config file with key " - + minimumResourceTypes + ", ignoring minimum specified using " - + arr[1]); - } - } else { - minimum = conf.getLong(arr[1], Long.parseLong(arr[2])); - } - ResourceInformation ri = res.get(arr[3]); - ri.setMinimumAllocation(minimum); - } - } - - private static void setMaximumAllocationForMandatoryResources( - Map<String, ResourceInformation> res, Configuration conf) { - String[][] resourceTypesKeys = { - {ResourceInformation.MEMORY_MB.getName(), - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), - ResourceInformation.MEMORY_MB.getName()}, - {ResourceInformation.VCORES.getName(), - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES), - ResourceInformation.VCORES.getName()}}; - for (String[] arr : resourceTypesKeys) { - String resourceTypesKey = - YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION; - long maximumResourceTypes = conf.getLong(resourceTypesKey, -1); - long maximumConf = conf.getLong(arr[1], -1); - long maximum; - if (maximumResourceTypes != -1) { - maximum = maximumResourceTypes; - if (maximumConf != -1) { - LOG.warn("Using maximum allocation for memory specified in " - + "resource-types config file with key " - + maximumResourceTypes + ", ignoring maximum specified using " - + arr[1]); - } - } else { - maximum = conf.getLong(arr[1], Long.parseLong(arr[2])); - } - ResourceInformation ri = res.get(arr[3]); - ri.setMaximumAllocation(maximum); + ResourceInformation mem = res.get(ResourceInformation.MEMORY_MB.getName()); + mem.setMinimumAllocation(getAllocation(conf, + YarnConfiguration.RESOURCE_TYPES + "." + + mem.getName() + MINIMUM_ALLOCATION, + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); + mem.setMaximumAllocation(getAllocation(conf, + YarnConfiguration.RESOURCE_TYPES + "." + + mem.getName() + MAXIMUM_ALLOCATION, + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); + + ResourceInformation cpu = res.get(ResourceInformation.VCORES.getName()); + + cpu.setMinimumAllocation(getAllocation(conf, + YarnConfiguration.RESOURCE_TYPES + "." + + cpu.getName() + MINIMUM_ALLOCATION, + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES)); + cpu.setMaximumAllocation(getAllocation(conf, + YarnConfiguration.RESOURCE_TYPES + "." + + cpu.getName() + MAXIMUM_ALLOCATION, + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)); + } + + private static long getAllocation(Configuration conf, + String resourceTypesKey, String schedulerKey, long schedulerDefault) { + long value = conf.getLong(resourceTypesKey, -1L); + if (value == -1) { + LOG.debug("Mandatory Resource '" + resourceTypesKey + "' is not " + + "configured in resource-types config file. Setting allocation " + + "specified using '" + schedulerKey + "'"); + value = conf.getLong(schedulerKey, schedulerDefault); } + return value; } @VisibleForTesting @@ -275,8 +245,7 @@ public class ResourceUtils { checkMandatoryResources(resourceInformationMap); addMandatoryResources(resourceInformationMap); - setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); - setMaximumAllocationForMandatoryResources(resourceInformationMap, conf); + setAllocationForMandatoryResources(resourceInformationMap, conf); initializeResourcesFromResourceInformationMap(resourceInformationMap); } @@ -473,8 +442,7 @@ public class ResourceUtils { conf); addMandatoryResources(nodeResources); checkMandatoryResources(nodeResources); - setMinimumAllocationForMandatoryResources(nodeResources, conf); - setMaximumAllocationForMandatoryResources(nodeResources, conf); + setAllocationForMandatoryResources(nodeResources, conf); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); initializedNodeResources = true; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
