Repository: hadoop Updated Branches: refs/heads/branch-2 933fad2e1 -> 8f533ca76
HDFS-11960. Successfully closed files can stay under-replicated. Contributed by Kihwal Lee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f533ca7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f533ca7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f533ca7 Branch: refs/heads/branch-2 Commit: 8f533ca76b3f3592dc8ec6e318382a54becf10f5 Parents: 933fad2 Author: Kihwal Lee <kih...@apache.org> Authored: Tue Jun 20 09:16:37 2017 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Tue Jun 20 09:16:37 2017 -0500 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 5 ++- .../blockmanagement/TestPendingReplication.java | 45 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f533ca7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5fe285e..7580be8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3354,8 +3354,9 @@ public class BlockManager implements BlockStatsMXBean { // Modify the blocks->datanode map and node's map. // BlockInfo storedBlock = getStoredBlock(block); - if (storedBlock != null) { - pendingReplications.decrement(getStoredBlock(block), node); + if (storedBlock != null && + block.getGenerationStamp() == storedBlock.getGenerationStamp()) { + pendingReplications.decrement(storedBlock, node); } processAndHandleReportedBlock(storageInfo, block, ReplicaState.FINALIZED, delHintNode); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f533ca7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index afda6e8..b77dbc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -209,6 +209,8 @@ public class TestPendingReplication { // Place into blocksmap with GenerationStamp = 1 blockInfo.setGenerationStamp(1); blocksMap.addBlockCollection(blockInfo, bc); + //Save it for later. + BlockInfo storedBlock = blockInfo; assertEquals("Size of pendingReplications ", 1, pendingReplications.size()); @@ -255,6 +257,49 @@ public class TestPendingReplication { // Verify size of neededReplications is exactly 1. assertEquals("size of neededReplications is 1 ", 1, neededReplications.size()); + + // Verify HDFS-11960 + // Stop the replication/redundancy monitor + BlockManagerTestUtil.stopReplicationThread(blkManager); + pendingReplications.clear(); + // Pick a real node + DatanodeDescriptor desc[] = { blkManager.getDatanodeManager(). + getDatanodes().iterator().next() }; + + // Add a stored block to the pendingReconstruction. + pendingReplications.increment(storedBlock, desc); + assertEquals("Size of pendingReplications ", 1, + pendingReplications.size()); + + // A received IBR processing calls addBlock(). If the gen stamp in the + // report is not the same, it should stay in pending. + fsn.writeLock(); + try { + // Use a wrong gen stamp. + blkManager.addBlock(desc[0].getStorageInfos()[0], + new Block(1, 1, 0), null); + } finally { + fsn.writeUnlock(); + } + + // The block should still be pending + assertEquals("Size of pendingReplications ", 1, + pendingReplications.size()); + + // A block report with the correct gen stamp should remove the record + // from the pending queue. + fsn.writeLock(); + try { + blkManager.addBlock(desc[0].getStorageInfos()[0], + new Block(1, 1, 1), null); + } finally { + fsn.writeUnlock(); + } + + // The pending queue should be empty. + assertEquals("Size of pendingReplications ", 0, + pendingReplications.size()); + } finally { if (cluster != null) { cluster.shutdown(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
