Repository: hadoop Updated Branches: refs/heads/trunk ac1aa6c81 -> 1c4951a7a
HDFS-8056. Decommissioned dead nodes should continue to be counted as dead after NN restart. (mingma) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c4951a7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c4951a7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c4951a7 Branch: refs/heads/trunk Commit: 1c4951a7a09433fbbcfe26f243d6c2d8043c71bb Parents: ac1aa6c Author: Ming Ma <min...@apache.org> Authored: Thu Nov 19 10:04:01 2015 -0800 Committer: Ming Ma <min...@apache.org> Committed: Thu Nov 19 10:04:01 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/DatanodeManager.java | 5 ++- .../apache/hadoop/hdfs/TestDecommission.java | 35 ++++++++++++++++++++ .../blockmanagement/TestHostFileManager.java | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c4951a7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d857c58..04bb1e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1657,6 +1657,9 @@ Release 2.8.0 - UNRELEASED HDFS-9252. Change TestFileTruncate to use FsDatasetTestUtils to get block file size and genstamp. (Lei (Eddy) Xu via cmccabe) + HDFS-8056. Decommissioned dead nodes should continue to be counted as dead + after NN restart. (mingma) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c4951a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 3406cf4..d35b237 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1272,7 +1272,7 @@ public class DatanodeManager { if (listDeadNodes) { for (InetSocketAddress addr : includedNodes) { - if (foundNodes.matchedBy(addr) || excludedNodes.match(addr)) { + if (foundNodes.matchedBy(addr)) { continue; } // The remaining nodes are ones that are referenced by the hosts @@ -1289,6 +1289,9 @@ public class DatanodeManager { addr.getPort() == 0 ? defaultXferPort : addr.getPort(), defaultInfoPort, defaultInfoSecurePort, defaultIpcPort)); setDatanodeDead(dn); + if (excludedNodes.match(addr)) { + dn.setDecommissioned(); + } nodes.add(dn); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c4951a7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index d648bca..0b70e24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -924,6 +924,41 @@ public class TestDecommission { } /** + * Tests dead node count after restart of namenode + **/ + @Test(timeout=360000) + public void testDeadNodeCountAfterNamenodeRestart()throws Exception { + LOG.info("Starting test testDeadNodeCountAfterNamenodeRestart"); + int numNamenodes = 1; + int numDatanodes = 2; + + startCluster(numNamenodes, numDatanodes, conf); + + DFSClient client = getDfsClient(cluster.getNameNode(), conf); + DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); + DatanodeInfo excludedDatanode = info[0]; + String excludedDatanodeName = info[0].getXferAddr(); + + writeConfigFile(hostsFile, new ArrayList<String>(Arrays.asList( + excludedDatanodeName, info[1].getXferAddr()))); + decommissionNode(0, excludedDatanode.getDatanodeUuid(), null, + AdminStates.DECOMMISSIONED); + + cluster.stopDataNode(excludedDatanodeName); + DFSTestUtil.waitForDatanodeState( + cluster, excludedDatanode.getDatanodeUuid(), false, 20000); + + //Restart the namenode + cluster.restartNameNode(); + + assertEquals("There should be one node alive", 1, + client.datanodeReport(DatanodeReportType.LIVE).length); + assertEquals("There should be one node dead", 1, + client.datanodeReport(DatanodeReportType.DEAD).length); + cluster.shutdown(); + } + + /** * Test using a "registration name" in a host include file. * * Registration names are DataNode names specified in the configuration by http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c4951a7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java index c65b580..6f17040 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java @@ -151,7 +151,7 @@ public class TestHostFileManager { Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants .DatanodeReportType.DEAD).size()); excludedNodes.add(entry("127.0.0.3")); - Assert.assertEquals(0, dm.getDatanodeListForReport(HdfsConstants + Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants .DatanodeReportType.DEAD).size()); } }
