YARN-4611. Fix scheduler load simulator to support multi-layer network location. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1708a4cd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1708a4cd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1708a4cd Branch: refs/heads/YARN-1011 Commit: 1708a4cd2377c56e2cb5738720b7eaf10baf13c8 Parents: 890a2eb Author: Xuan <xg...@apache.org> Authored: Wed Jan 20 15:41:50 2016 -0800 Committer: Xuan <xg...@apache.org> Committed: Wed Jan 20 15:41:50 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java | 11 +++++++++-- .../hadoop/yarn/sls/nodemanager/TestNMSimulator.java | 2 +- .../org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java | 5 +++++ hadoop-yarn-project/CHANGES.txt | 3 +++ 4 files changed, 18 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java index 922370c..d950aee 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java @@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.tools.rumen.JobTraceReader; import org.apache.hadoop.tools.rumen.LoggedJob; import org.apache.hadoop.tools.rumen.LoggedTask; @@ -43,9 +44,15 @@ import org.codehaus.jackson.map.ObjectMapper; @Unstable public class SLSUtils { + // hostname includes the network path and the host name. for example + // "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar". + // the function returns two Strings, the first element is the network + // location without "/", the second element is the host name. for example, + // {"default-rack", "hostFoo"} or "coreSwitchA/TORSwitchB", "hostBar" public static String[] getRackHostName(String hostname) { - hostname = hostname.substring(1); - return hostname.split("/"); + NodeBase node = new NodeBase(hostname); + return new String[] {node.getNetworkLocation().substring(1), + node.getName()}; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java index 2381549..f9a3932 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java @@ -51,7 +51,7 @@ public class TestNMSimulator { public void testNMSimulator() throws Exception { // Register one node NMSimulator node1 = new NMSimulator(); - node1.init("rack1/node1", GB * 10, 10, 0, 1000, rm); + node1.init("/rack1/node1", GB * 10, 10, 0, 1000, rm); node1.middleStep(); int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java index 498615a..f4eda67 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java @@ -29,6 +29,11 @@ public class TestSLSUtils { String rackHostname[] = SLSUtils.getRackHostName(str); Assert.assertEquals(rackHostname[0], "rack1"); Assert.assertEquals(rackHostname[1], "node1"); + + str = "/rackA/rackB/node1"; + rackHostname = SLSUtils.getRackHostName(str); + Assert.assertEquals(rackHostname[0], "rackA/rackB"); + Assert.assertEquals(rackHostname[1], "node1"); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ee78d22..212ffa1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -131,6 +131,9 @@ Release 2.9.0 - UNRELEASED YARN-3446. FairScheduler headroom calculation should exclude nodes in the blacklist. (Zhihai Xu via kasha) + YARN-4611. Fix scheduler load simulator to support multi-layer network + location. (Ming Ma via xgong) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES