YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7b41f1f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7b41f1f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7b41f1f Branch: refs/heads/YARN-8200 Commit: c7b41f1f69e247deab938a0ff61177b7ba47f095 Parents: 13435f1 Author: Sunil G <sun...@apache.org> Authored: Wed Dec 13 22:00:07 2017 +0530 Committer: Konstantin V Shvachko <kshvac...@linkedin.com> Committed: Wed May 2 13:19:58 2018 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/util/resource/ResourceUtils.java | 15 ++++++--------- .../hadoop/yarn/util/resource/TestResourceUtils.java | 5 ++++- .../resources/resource-types/node-resources-2.xml | 5 +++++ .../resources/resource-types/resource-types-4.xml | 7 ++++++- 4 files changed, 21 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7b41f1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index abf58a6..65eb5a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -461,21 +461,18 @@ public class ResourceUtils { for (Map.Entry<String, String> entry : conf) { String key = entry.getKey(); String value = entry.getValue(); - - if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { - addResourceInformation(key, value, nodeResources); - } + addResourceTypeInformation(key, value, nodeResources); } return nodeResources; } - private static void addResourceInformation(String prop, String value, + private static void addResourceTypeInformation(String prop, String value, Map<String, ResourceInformation> nodeResources) { - String[] parts = prop.split("\\."); - LOG.info("Found resource entry " + prop); - if (parts.length == 4) { - String resourceType = parts[3]; + if (prop.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { + LOG.info("Found resource entry " + prop); + String resourceType = prop.substring( + YarnConfiguration.NM_RESOURCES_PREFIX.length()); if (!nodeResources.containsKey(resourceType)) { nodeResources .put(resourceType, ResourceInformation.newInstance(resourceType)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7b41f1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 80555ca..b511705 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -124,9 +124,10 @@ public class TestResourceUtils { new ResourceFileInformation("resource-types-3.xml", 3); testFile3.resourceNameUnitsMap.put("resource2", ""); ResourceFileInformation testFile4 = - new ResourceFileInformation("resource-types-4.xml", 4); + new ResourceFileInformation("resource-types-4.xml", 5); testFile4.resourceNameUnitsMap.put("resource1", "G"); testFile4.resourceNameUnitsMap.put("resource2", "m"); + testFile4.resourceNameUnitsMap.put("yarn.io/gpu", ""); ResourceFileInformation[] tests = {testFile1, testFile2, testFile3, testFile4}; @@ -292,6 +293,8 @@ public class TestResourceUtils { ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2L)); + test3Resources.setResourceInformation("yarn.io/gpu", + ResourceInformation.newInstance("yarn.io/gpu", "", 1)); testRun.put("node-resources-2.xml", test3Resources); for (Map.Entry<String, Resource> entry : testRun.entrySet()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7b41f1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml index 9d9b3dc..382d5dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml @@ -36,4 +36,9 @@ limitations under the License. See accompanying LICENSE file. <value>2m</value> </property> + <property> + <name>yarn.nodemanager.resource-type.yarn.io/gpu</name> + <value>1</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7b41f1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml index c84316a..ea8d2bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml @@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file. <property> <name>yarn.resource-types</name> - <value>resource1,resource2</value> + <value>resource1,resource2,yarn.io/gpu</value> </property> <property> @@ -31,4 +31,9 @@ limitations under the License. See accompanying LICENSE file. <value>m</value> </property> + <property> + <name>yarn.resource-types.yarn.io/gpu.units</name> + <value></value> + </property> + </configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org