Repository: hadoop Updated Branches: refs/heads/trunk 368743140 -> abae63caf
YARN-2699. Fixed a bug in CommonNodeLabelsManager that caused tests to fail when using ephemeral ports on NodeIDs. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abae63ca Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abae63ca Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abae63ca Branch: refs/heads/trunk Commit: abae63caf9c53b404f2f2db7d482555484eaeaf8 Parents: 3687431 Author: Vinod Kumar Vavilapalli <[email protected]> Authored: Fri Oct 17 08:58:08 2014 -0700 Committer: Vinod Kumar Vavilapalli <[email protected]> Committed: Fri Oct 17 08:58:08 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodelabels/CommonNodeLabelsManager.java | 37 ++++++++++---------- .../nodelabels/RMNodeLabelsManager.java | 12 +++++-- .../nodelabels/TestRMNodeLabelsManager.java | 8 +++++ 4 files changed, 40 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e8b0ef5..5056470 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -594,6 +594,9 @@ Release 2.6.0 - UNRELEASED tracking per label when a host runs multiple node-managers. (Wangda Tan via vinodkv) + YARN-2699. Fixed a bug in CommonNodeLabelsManager that caused tests to fail + when using ephemeral ports on NodeIDs. (Wangda Tan via vinodkv) + BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS YARN-1707. Introduce APIs to add/remove/resize queues in the http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 511b5ee..8bb88f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -299,14 +299,14 @@ public class CommonNodeLabelsManager extends AbstractService { for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) { NodeId nodeId = entry.getKey(); Set<String> labels = entry.getValue(); - - createNodeIfNonExisted(entry.getKey()); - + + createHostIfNonExisted(nodeId.getHost()); if (nodeId.getPort() == WILDCARD_PORT) { Host host = nodeCollections.get(nodeId.getHost()); host.labels.addAll(labels); newNMToLabels.put(nodeId, host.labels); } else { + createNodeIfNonExisted(nodeId); Node nm = getNMInNodeSet(nodeId); if (nm.labels == null) { nm.labels = new HashSet<String>(); @@ -534,21 +534,21 @@ public class CommonNodeLabelsManager extends AbstractService { @SuppressWarnings("unchecked") protected void internalReplaceLabelsOnNode( - Map<NodeId, Set<String>> replaceLabelsToNode) { + Map<NodeId, Set<String>> replaceLabelsToNode) throws IOException { // do replace labels to nodes Map<NodeId, Set<String>> newNMToLabels = new HashMap<NodeId, Set<String>>(); for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) { NodeId nodeId = entry.getKey(); Set<String> labels = entry.getValue(); - // update nodeCollections - createNodeIfNonExisted(entry.getKey()); + createHostIfNonExisted(nodeId.getHost()); if (nodeId.getPort() == WILDCARD_PORT) { Host host = nodeCollections.get(nodeId.getHost()); host.labels.clear(); host.labels.addAll(labels); newNMToLabels.put(nodeId, host.labels); } else { + createNodeIfNonExisted(nodeId); Node nm = getNMInNodeSet(nodeId); if (nm.labels == null) { nm.labels = new HashSet<String>(); @@ -672,10 +672,6 @@ public class CommonNodeLabelsManager extends AbstractService { protected Node getNMInNodeSet(NodeId nodeId, Map<String, Host> map, boolean checkRunning) { - if (WILDCARD_PORT == nodeId.getPort()) { - return null; - } - Host host = map.get(nodeId.getHost()); if (null == host) { return null; @@ -707,17 +703,22 @@ public class CommonNodeLabelsManager extends AbstractService { } } - protected void createNodeIfNonExisted(NodeId nodeId) { + protected void createNodeIfNonExisted(NodeId nodeId) throws IOException { Host host = nodeCollections.get(nodeId.getHost()); if (null == host) { - host = new Host(); - nodeCollections.put(nodeId.getHost(), host); + throw new IOException("Should create host before creating node."); } - if (nodeId.getPort() != WILDCARD_PORT) { - Node nm = host.nms.get(nodeId); - if (null == nm) { - host.nms.put(nodeId, new Node()); - } + Node nm = host.nms.get(nodeId); + if (null == nm) { + host.nms.put(nodeId, new Node()); + } + } + + protected void createHostIfNonExisted(String hostName) { + Host host = nodeCollections.get(hostName); + if (null == host) { + host = new Host(); + nodeCollections.put(hostName, host); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 1d7f6f1..ba1727c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -181,7 +181,15 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // save if we have a node before Map<String, Host> before = cloneNodeMap(ImmutableSet.of(nodeId)); - createNodeIfNonExisted(nodeId); + createHostIfNonExisted(nodeId.getHost()); + try { + createNodeIfNonExisted(nodeId); + } catch (IOException e) { + LOG.error("This shouldn't happen, cannot get host in nodeCollection" + + " associated to the node being activated"); + return; + } + Node nm = getNMInNodeSet(nodeId); nm.resource = resource; nm.running = true; @@ -220,7 +228,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } } - public void updateNodeResource(NodeId node, Resource newResource) { + public void updateNodeResource(NodeId node, Resource newResource) throws IOException { deactivateNode(node); activateNode(node, newResource); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 81eead9..1fbe968 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -96,6 +96,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Assert.assertEquals(mgr.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, null), Resources.add(SMALL_RESOURCE, LARGE_NODE)); } + + @Test(timeout = 5000) + public void testActivateNodeManagerWithZeroPort() throws Exception { + // active two NM, one is zero port , another is non-zero port. no exception + // should be raised + mgr.activateNode(NodeId.newInstance("n1", 0), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 2), LARGE_NODE); + } @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000)
