Author: vinodkv Date: Wed Aug 28 05:56:21 2013 New Revision: 1518082 URL: http://svn.apache.org/r1518082 Log: YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza) svn merge --ignore-ancestry -c 1517083 ../../trunk/
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1518082&r1=1518081&r2=1518082&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Wed Aug 28 05:56:21 2013 @@ -19,6 +19,8 @@ Release 2.1.1-beta - UNRELEASED YARN-942. In Fair Scheduler documentation, inconsistency on which properties have prefix (Akira Ajisaka via Sandy Ryza) + YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza) + YARN-1083. Changed ResourceManager to fail when the expiry interval is less than the configured node-heartbeat interval. (Zhijie Shen via vinodkv) Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1518082&r1=1518081&r2=1518082&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java Wed Aug 28 05:56:21 2013 @@ -21,11 +21,14 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -40,9 +43,12 @@ import org.apache.hadoop.yarn.util.Conve @Private @Unstable public class NodeCLI extends YarnCLI { - private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%18s" + + private static final String NODES_PATTERN = "%16s\t%15s\t%17s\t%18s" + System.getProperty("line.separator"); + private static final String NODE_STATE_CMD = "states"; + private static final String NODE_ALL = "all"; + public static void main(String[] args) throws Exception { NodeCLI cli = new NodeCLI(); cli.setSysOutPrintStream(System.out); @@ -57,7 +63,18 @@ public class NodeCLI extends YarnCLI { Options opts = new Options(); opts.addOption(STATUS_CMD, true, "Prints the status report of the node."); - opts.addOption(LIST_CMD, false, "Lists all the nodes in the RUNNING state."); + opts.addOption(LIST_CMD, false, "List all running nodes. " + + "Supports optional use of --states to filter nodes " + + "based on node state, all --all to list all nodes."); + Option nodeStateOpt = new Option(NODE_STATE_CMD, true, + "Works with -list to filter nodes based on their states."); + nodeStateOpt.setValueSeparator(','); + nodeStateOpt.setArgs(Option.UNLIMITED_VALUES); + nodeStateOpt.setArgName("Comma-separated list of node states"); + opts.addOption(nodeStateOpt); + Option allOpt = new Option(NODE_ALL, false, + "Works with -list to list all nodes."); + opts.addOption(allOpt); CommandLine cliParser = new GnuParser().parse(opts, args); int exitCode = -1; @@ -68,7 +85,24 @@ public class NodeCLI extends YarnCLI { } printNodeStatus(cliParser.getOptionValue("status")); } else if (cliParser.hasOption("list")) { - listClusterNodes(); + Set<NodeState> nodeStates = new HashSet<NodeState>(); + if (cliParser.hasOption(NODE_ALL)) { + for (NodeState state : NodeState.values()) { + nodeStates.add(state); + } + } else if (cliParser.hasOption(NODE_STATE_CMD)) { + String[] types = cliParser.getOptionValues(NODE_STATE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + nodeStates.add(NodeState.valueOf(type.trim().toUpperCase())); + } + } + } + } else { + nodeStates.add(NodeState.RUNNING); + } + listClusterNodes(nodeStates); } else { syserr.println("Invalid Command Usage : "); printUsage(opts); @@ -86,14 +120,17 @@ public class NodeCLI extends YarnCLI { } /** - * Lists all the nodes present in the cluster + * Lists the nodes matching the given node states * + * @param nodeStates * @throws YarnException * @throws IOException */ - private void listClusterNodes() throws YarnException, IOException { + private void listClusterNodes(Set<NodeState> nodeStates) + throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); - List<NodeReport> nodesReport = client.getNodeReports(NodeState.RUNNING); + List<NodeReport> nodesReport = client.getNodeReports( + nodeStates.toArray(new NodeState[0])); writer.println("Total Nodes:" + nodesReport.size()); writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address", "Running-Containers"); Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1518082&r1=1518081&r2=1518082&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Wed Aug 28 05:56:21 2013 @@ -519,36 +519,239 @@ public class TestYarnCLI { @Test public void testListClusterNodes() throws Exception { + List<NodeReport> nodeReports = new ArrayList<NodeReport>(); + nodeReports.addAll(getNodeReports(1, NodeState.NEW)); + nodeReports.addAll(getNodeReports(2, NodeState.RUNNING)); + nodeReports.addAll(getNodeReports(1, NodeState.UNHEALTHY)); + nodeReports.addAll(getNodeReports(1, NodeState.DECOMMISSIONED)); + nodeReports.addAll(getNodeReports(1, NodeState.REBOOTED)); + nodeReports.addAll(getNodeReports(1, NodeState.LOST)); + NodeCLI cli = new NodeCLI(); - when(client.getNodeReports(NodeState.RUNNING)).thenReturn( - getNodeReports(3)); cli.setClient(client); cli.setSysOutPrintStream(sysOut); - int result = cli.run(new String[] { "-list" }); + + Set<NodeState> nodeStates = new HashSet<NodeState>(); + nodeStates.add(NodeState.NEW); + NodeState[] states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + int result = cli.run(new String[] { "-list", "--states", "NEW" }); assertEquals(0, result); - verify(client).getNodeReports(NodeState.RUNNING); + verify(client).getNodeReports(states); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.println("Total Nodes:3"); - pw.print(" Node-Id\tNode-State\tNode-Http-Address\t"); + pw.println("Total Nodes:1"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); pw.println("Running-Containers"); - pw.print(" host0:0\t RUNNING\t host1:8888"); - pw.println("\t 0"); - pw.print(" host1:0\t RUNNING\t host1:8888"); - pw.println("\t 0"); - pw.print(" host2:0\t RUNNING\t host1:8888"); + pw.print(" host0:0\t NEW\t host1:8888"); pw.println("\t 0"); pw.close(); String nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.RUNNING); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", "RUNNING" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:2"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.print(" host1:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + result = cli.run(new String[] { "-list" }); + assertEquals(0, result); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.UNHEALTHY); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", "UNHEALTHY" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:1"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t UNHEALTHY\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.DECOMMISSIONED); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", "DECOMMISSIONED" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:1"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t DECOMMISSIONED\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.REBOOTED); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", "REBOOTED" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:1"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t REBOOTED\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.LOST); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", "LOST" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:1"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t LOST\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + nodeStates.add(NodeState.NEW); + nodeStates.add(NodeState.RUNNING); + nodeStates.add(NodeState.LOST); + nodeStates.add(NodeState.REBOOTED); + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--states", + "NEW,RUNNING,LOST,REBOOTED" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:5"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t NEW\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.print(" host1:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t REBOOTED\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t LOST\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + nodeStates.clear(); + for (NodeState s : NodeState.values()) { + nodeStates.add(s); + } + states = nodeStates.toArray(new NodeState[0]); + when(client.getNodeReports(states)) + .thenReturn(getNodeReports(nodeReports, nodeStates)); + result = cli.run(new String[] { "-list", "--all" }); + assertEquals(0, result); + verify(client).getNodeReports(states); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:7"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Running-Containers"); + pw.print(" host0:0\t NEW\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.print(" host1:0\t RUNNING\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t UNHEALTHY\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t DECOMMISSIONED\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t REBOOTED\t host1:8888"); + pw.println("\t 0"); + pw.print(" host0:0\t LOST\t host1:8888"); + pw.println("\t 0"); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt()); + } + + private List<NodeReport> getNodeReports( + List<NodeReport> nodeReports, + Set<NodeState> nodeStates) { + List<NodeReport> reports = new ArrayList<NodeReport>(); + + for (NodeReport nodeReport : nodeReports) { + if (nodeStates.contains(nodeReport.getNodeState())) { + reports.add(nodeReport); + } + } + return reports; } @Test public void testNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); NodeCLI cli = new NodeCLI(); - when(client.getNodeReports()).thenReturn(getNodeReports(3)); + when(client.getNodeReports()).thenReturn( + getNodeReports(3, NodeState.RUNNING)); cli.setClient(client); cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); @@ -580,7 +783,8 @@ public class TestYarnCLI { public void testAbsentNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("Absenthost0", 0); NodeCLI cli = new NodeCLI(); - when(client.getNodeReports()).thenReturn(getNodeReports(0)); + when(client.getNodeReports()).thenReturn( + getNodeReports(0, NodeState.RUNNING)); cli.setClient(client); cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); @@ -608,12 +812,12 @@ public class TestYarnCLI { verify(sysErr).println("Invalid Command Usage : "); } - private List<NodeReport> getNodeReports(int noOfNodes) { + private List<NodeReport> getNodeReports(int noOfNodes, NodeState state) { List<NodeReport> nodeReports = new ArrayList<NodeReport>(); for (int i = 0; i < noOfNodes; i++) { NodeReport nodeReport = NodeReport.newInstance(NodeId - .newInstance("host" + i, 0), NodeState.RUNNING, "host" + 1 + ":8888", + .newInstance("host" + i, 0), state, "host" + 1 + ":8888", "rack1", Records.newRecord(Resource.class), Records .newRecord(Resource.class), 0, "", 0); nodeReports.add(nodeReport);