Repository: hadoop Updated Branches: refs/heads/branch-2.8 7a5aaa789 -> ded91992a
HDFS-10987. Make Decommission less expensive when lot of blocks present. Contributed by Brahma Reddy Battula. (cherry picked from commit c5a13037048eb1e3b5a500aeec0e2e953e7d509a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ded91992 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ded91992 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ded91992 Branch: refs/heads/branch-2.8 Commit: ded91992adc08c5ac5cff00abcb9f05c148d8daa Parents: 7a5aaa7 Author: Kihwal Lee <kih...@apache.org> Authored: Thu Oct 13 14:57:44 2016 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Thu Oct 13 14:57:44 2016 -0500 ---------------------------------------------------------------------- .../blockmanagement/DecommissionManager.java | 29 +++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ded91992/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 073332b..be4771d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -355,6 +355,10 @@ public class DecommissionManager { */ private int numBlocksChecked = 0; /** + * The number of blocks checked after (re)holding lock. + */ + private int numBlocksCheckedPerLock = 0; + /** * The number of nodes that have been checked on this tick. Used for * testing. */ @@ -392,6 +396,7 @@ public class DecommissionManager { } // Reset the checked count at beginning of each iteration numBlocksChecked = 0; + numBlocksCheckedPerLock = 0; numNodesChecked = 0; // Check decom progress namesystem.writeLock(); @@ -426,7 +431,8 @@ public class DecommissionManager { while (it.hasNext() && !exceededNumBlocksPerCheck() - && !exceededNumNodesPerCheck()) { + && !exceededNumNodesPerCheck() + && namesystem.isRunning()) { numNodesChecked++; final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>> entry = it.next(); @@ -544,7 +550,28 @@ public class DecommissionManager { int decommissionOnlyReplicas = 0; int underReplicatedInOpenFiles = 0; while (it.hasNext()) { + if (insufficientlyReplicated == null + && numBlocksCheckedPerLock >= numBlocksPerCheck) { + // During fullscan insufficientlyReplicated will NOT be null, iterator + // will be DN's iterator. So should not yield lock, otherwise + // ConcurrentModificationException could occur. + // Once the fullscan done, iterator will be a copy. So can yield the + // lock. + // Yielding is required in case of block number is greater than the + // configured per-iteration-limit. + namesystem.writeUnlock(); + try { + LOG.debug("Yielded lock during decommission check"); + Thread.sleep(0, 500); + } catch (InterruptedException ignored) { + return; + } + // reset + numBlocksCheckedPerLock = 0; + namesystem.writeLock(); + } numBlocksChecked++; + numBlocksCheckedPerLock++; final BlockInfo block = it.next(); // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
