Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 3e223d5dc -> d389afb3d
YARN-4940. yarn node -list -all failed if RM start with decommissioned node.
Contributed by sandflee
(cherry picked from commit 69f3d428d5c3ab0c79cacffc22b1f59408622ae7)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d389afb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d389afb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d389afb3
Branch: refs/heads/branch-2.8
Commit: d389afb3da4680c0baf1b8c711b28c87bc9994a6
Parents: 3e223d5
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Apr 15 20:45:20 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Apr 15 20:45:20 2016 +0000
----------------------------------------------------------------------
.../resourcemanager/NodesListManager.java | 36 ++------------
.../resourcemanager/rmnode/RMNodeImpl.java | 4 +-
.../resourcemanager/TestClientRMService.java | 49 +++++++++++++++++++-
.../resourcemanager/TestRMNodeTransitions.java | 4 +-
4 files changed, 55 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d389afb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index 89027b1..40b7731 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -162,7 +162,7 @@ public class NodesListManager extends CompositeService
implements
private void setDecomissionedNMs() {
Set<String> excludeList = hostsReader.getExcludedHosts();
for (final String host : excludeList) {
- UnknownNodeId nodeId = new UnknownNodeId(host);
+ NodeId nodeId = createUnknownNodeId(host);
RMNodeImpl rmNode = new RMNodeImpl(nodeId,
rmContext, host, -1, -1, new UnknownNode(host), null, null);
rmContext.getInactiveRMNodes().put(nodeId, rmNode);
@@ -429,38 +429,8 @@ public class NodesListManager extends CompositeService
implements
* A NodeId instance needed upon startup for populating inactive nodes Map.
* It only knows the hostname/ip and marks the port to -1 or invalid.
*/
- public static class UnknownNodeId extends NodeId {
-
- private String host;
-
- public UnknownNodeId(String host) {
- this.host = host;
- }
-
- @Override
- public String getHost() {
- return this.host;
- }
-
- @Override
- protected void setHost(String hst) {
-
- }
-
- @Override
- public int getPort() {
- return -1;
- }
-
- @Override
- protected void setPort(int port) {
-
- }
-
- @Override
- protected void build() {
-
- }
+ public static NodeId createUnknownNodeId(String host) {
+ return NodeId.newInstance(host, -1);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d389afb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 607e46c..b02994a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -772,8 +772,8 @@ public class RMNodeImpl implements RMNode,
EventHandler<RMNodeEvent> {
if (previousRMNode != null) {
rmNode.updateMetricsForRejoinedNode(previousRMNode.getState());
} else {
- NodesListManager.UnknownNodeId unknownNodeId =
- new NodesListManager.UnknownNodeId(nodeId.getHost());
+ NodeId unknownNodeId =
+ NodesListManager.createUnknownNodeId(nodeId.getHost());
previousRMNode =
rmNode.context.getInactiveRMNodes().remove(unknownNodeId);
if (previousRMNode != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d389afb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 6fccc63..ecd8966 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -28,6 +28,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -247,7 +249,7 @@ public class TestClientRMService {
Assert.assertTrue(report.getNodeLabels() != null
&& report.getNodeLabels().isEmpty());
}
-
+
rpc.stopProxy(client, conf);
rm.close();
}
@@ -1554,4 +1556,49 @@ public class TestClientRMService {
rm.stop();
}
+
+ private void createExcludeFile(String filename) throws IOException {
+ File file = new File(filename);
+ if (file.exists()) {
+ file.delete();
+ }
+
+ FileOutputStream out = new FileOutputStream(file);
+ out.write("decommisssionedHost".getBytes());
+ out.close();
+ }
+
+ @Test
+ public void testRMStartWithDecommissionedNode() throws Exception {
+ String excludeFile = "excludeFile";
+ createExcludeFile(excludeFile);
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ excludeFile);
+ MockRM rm = new MockRM(conf) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager,
this.queueACLsManager,
+ this.getRMContext().getRMDelegationTokenSecretManager());
+ };
+ };
+ rm.start();
+
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ ApplicationClientProtocol client =
+ (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+ // Make call
+ GetClusterNodesRequest request =
+ GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
+ List<NodeReport> nodeReports =
client.getClusterNodes(request).getNodeReports();
+ Assert.assertEquals(1, nodeReports.size());
+
+ rm.stop();
+ rpc.stopProxy(client, conf);
+ new File(excludeFile).delete();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d389afb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 6ba360b..7c03574 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -690,8 +690,8 @@ public class TestRMNodeTransitions {
@Test
public void testUnknownNodeId() {
- NodesListManager.UnknownNodeId nodeId =
- new NodesListManager.UnknownNodeId("host1");
+ NodeId nodeId =
+ NodesListManager.createUnknownNodeId("host1");
RMNodeImpl node =
new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
rmContext.getInactiveRMNodes().putIfAbsent(nodeId,node);
