HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. Contributed by Walter Su
(cherry picked from commit 4e1f2eb3955a97a70cf127dc97ae49201a90f5e0) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a28c6a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a28c6a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a28c6a3 Branch: refs/heads/sjlee/hdfs-merge Commit: 5a28c6a37cab5f1061b6ed9536341da537d51b5a Parents: b054cb6 Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Thu May 7 11:36:35 2015 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Thu Aug 13 16:32:37 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a28c6a3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index e5d97d1..e271d55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1815,7 +1815,7 @@ public class BlockManager { return !node.hasStaleStorages(); } - if (storageInfo.numBlocks() == 0) { + if (storageInfo.getBlockReportCount() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. processFirstBlockReport(storageInfo, newReport); @@ -2038,7 +2038,7 @@ public class BlockManager { final BlockListAsLongs report) throws IOException { if (report == null) return; assert (namesystem.hasWriteLock()); - assert (storageInfo.numBlocks() == 0); + assert (storageInfo.getBlockReportCount() == 0); BlockReportIterator itBR = report.getBlockReportIterator(); while(itBR.hasNext()) { @@ -2451,14 +2451,14 @@ public class BlockManager { } // just add it - storageInfo.addBlock(storedBlock); + boolean added = storageInfo.addBlock(storedBlock); // Now check for completion of blocks and safe block count int numCurrentReplica = countLiveNodes(storedBlock); if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED && numCurrentReplica >= minReplication) { completeBlock(storedBlock.getBlockCollection(), storedBlock, false); - } else if (storedBlock.isComplete()) { + } else if (storedBlock.isComplete() && added) { // check whether safe replication is reached for the block // only complete blocks are counted towards that. // In the case that the block just became complete above, completeBlock()