Repository: hadoop Updated Branches: refs/heads/branch-2.7 49f983633 -> c1a6f4f2e
YARN-5837. NPE when getting node status of a decommissioned node after an RM restart. Contributed by Robert Kanter Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a6f4f2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a6f4f2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a6f4f2 Branch: refs/heads/branch-2.7 Commit: c1a6f4f2ee4ff470a6ea37275b8206b2955832c0 Parents: 49f9836 Author: Jason Lowe <jl...@apache.org> Authored: Fri Nov 4 22:41:11 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Fri Nov 4 22:41:11 2016 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/resourcemanager/NodesListManager.java | 4 +++- .../yarn/server/resourcemanager/MockNM.java | 4 ++++ .../server/resourcemanager/TestRMRestart.java | 19 +++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a6f4f2/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0beee1d..e45282c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -59,6 +59,9 @@ Release 2.7.4 - UNRELEASED YARN-5001. Aggregated Logs root directory is created with wrong group if nonexistent (Haibo Chen via jlowe) + YARN-5837. NPE when getting node status of a decommissioned node after an + RM restart (Robert Kanter via jlowe) + Release 2.7.3 - 2016-08-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a6f4f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index 187398b..02d11ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -32,6 +32,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.HostsFileReader; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -142,7 +143,8 @@ public class NodesListManager extends AbstractService implements for (final String host : excludeList) { NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, - rmContext, host, -1, -1, new UnknownNode(host), null, null); + rmContext, host, -1, -1, new UnknownNode(host), + Resource.newInstance(0, 0), "unknown"); rmContext.getInactiveRMNodes().put(nodeId.getHost(), rmNode); rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a6f4f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index cbd83c7..c7147a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -231,4 +231,8 @@ public class MockNM { public int getvCores() { return vCores; } + + public String getVersion() { + return version; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a6f4f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 469135b..6dfed02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -98,6 +99,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -1856,6 +1858,9 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { rm1.start(); MockNM nm1 = rm1.registerNode("localhost:1234", 8000); MockNM nm2 = rm1.registerNode("host2:1234", 8000); + Resource expectedCapability = + Resource.newInstance(nm1.getMemory(), nm1.getvCores()); + String expectedVersion = nm1.getVersion(); Assert .assertEquals(0, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); @@ -1877,6 +1882,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { Assert .assertEquals(2, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); + verifyNodesAfterDecom(rm1, 2, expectedCapability, expectedVersion); rm1.stop(); rm1 = null; Assert @@ -1890,6 +1896,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { Assert .assertEquals(2, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); + verifyNodesAfterDecom(rm2, 2, Resource.newInstance(0, 0), "unknown"); } finally { if (rm1 != null) { rm1.stop(); @@ -1900,6 +1907,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { } } + private void verifyNodesAfterDecom(MockRM rm, int numNodes, + Resource expectedCapability, + String expectedVersion) { + ConcurrentMap<String, RMNode> inactiveRMNodes = + rm.getRMContext().getInactiveRMNodes(); + Assert.assertEquals(numNodes, inactiveRMNodes.size()); + for (RMNode rmNode : inactiveRMNodes.values()) { + Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); + Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); + } + } + // Test Delegation token is renewed synchronously so that recover events // can be processed before any other external incoming events, specifically // the ContainerFinished event on NM re-registraton. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
