Repository: hadoop Updated Branches: refs/heads/branch-2 c359e2a6a -> 988682ac2
HDFS-7409. Allow dead nodes to finish decommissioning if all files are fully replicated. (cherry picked from commit 765aecb4e127ebaf6c7b060a8b5cd40c6428e296) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/988682ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/988682ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/988682ac Branch: refs/heads/branch-2 Commit: 988682ac2957da0f34e9fe524d71c5931365fe95 Parents: c359e2a Author: Andrew Wang <[email protected]> Authored: Wed Nov 19 17:53:00 2014 -0800 Committer: Andrew Wang <[email protected]> Committed: Wed Nov 19 17:53:25 2014 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/blockmanagement/BlockManager.java | 18 +++++++++++++----- .../namenode/TestDecommissioningStatus.java | 13 ++++++++++--- 3 files changed, 26 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/988682ac/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 612d7d9..b1ccb0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -118,6 +118,9 @@ Release 2.7.0 - UNRELEASED HDFS-7398. Reset cached thread-local FSEditLogOp's on every FSEditLog#logEdit. (Gera Shegalov via cnauroth) + HDFS-7409. Allow dead nodes to finish decommissioning if all files are + fully replicated. (wang) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/988682ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index be38e46..e23f39a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3275,11 +3275,19 @@ public class BlockManager { } if (!status && !srcNode.isAlive) { - LOG.warn("srcNode " + srcNode + " is dead " + - "when decommission is in progress. Continue to mark " + - "it as decommission in progress. In that way, when it rejoins the " + - "cluster it can continue the decommission process."); - status = true; + updateState(); + if (pendingReplicationBlocksCount == 0 && + underReplicatedBlocksCount == 0) { + LOG.info("srcNode {} is dead and there are no under-replicated" + + " blocks or blocks pending replication. Marking as " + + "decommissioned."); + } else { + LOG.warn("srcNode " + srcNode + " is dead " + + "while decommission is in progress. Continuing to mark " + + "it as decommission in progress so when it rejoins the " + + "cluster it can continue the decommission process."); + status = true; + } } srcNode.decommissioningStatus.set(underReplicatedBlocks, http://git-wip-us.apache.org/repos/asf/hadoop/blob/988682ac/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java index c6312c0..a2b7b87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -359,8 +360,15 @@ public class TestDecommissioningStatus { BlockManagerTestUtil.checkDecommissionState(dm, dead.get(0)); // Verify that the DN remains in DECOMMISSION_INPROGRESS state. - assertTrue("the node is in decommissioned state ", - !dead.get(0).isDecommissioned()); + assertTrue("the node should be DECOMMISSION_IN_PROGRESSS", + dead.get(0).isDecommissionInProgress()); + + // Delete the under-replicated file, which should let the + // DECOMMISSION_IN_PROGRESS node become DECOMMISSIONED + cleanupFile(fileSys, f); + BlockManagerTestUtil.checkDecommissionState(dm, dead.get(0)); + assertTrue("the node should be decommissioned", + dead.get(0).isDecommissioned()); // Add the node back cluster.restartDataNode(dataNodeProperties, true); @@ -371,7 +379,6 @@ public class TestDecommissioningStatus { // make them available again. writeConfigFile(localFileSys, excludeFile, null); dm.refreshNodes(conf); - cleanupFile(fileSys, f); } /**
