Repository: hadoop Updated Branches: refs/heads/branch-2.7 1865cc5bd -> 62ce8f043
HDFS-11742. Improve balancer usability after HDFS-8818. Contributed by Kihwal Lee Updated CHANGES.txt (cherry picked from commit e229ffee64e8abd1df4d819b81151582ed3a16e2) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62ce8f04 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62ce8f04 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62ce8f04 Branch: refs/heads/branch-2.7 Commit: 62ce8f043065dfd6c98945cd2ce92b6825b16bef Parents: 1865cc5 Author: Kihwal Lee <kih...@apache.org> Authored: Fri Jul 21 10:09:32 2017 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Fri Jul 21 10:09:56 2017 -0500 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/balancer/Dispatcher.java | 35 +++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/62ce8f04/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8b1bf44..5713513 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -394,6 +394,8 @@ Release 2.7.4 - UNRELEASED HDFS-12177. NameNode exits due to setting BlockPlacementPolicy loglevel to Debug. (Jiandan Yang via Brahma Reddy Battula) + HDFS-11742. Improve balancer usability after HDFS-8818. (kihwal) + Release 2.7.3 - 2016-08-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/62ce8f04/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 75c32f9..b3e591e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -118,6 +118,7 @@ public class Dispatcher { /** The maximum number of concurrent blocks moves at a datanode */ private final int maxConcurrentMovesPerNode; + private final int maxMoverThreads; private final long getBlocksSize; private final long getBlocksMinBlockSize; @@ -131,11 +132,13 @@ public class Dispatcher { static class Allocator { private final int max; private int count = 0; + private int lotSize = 1; Allocator(int max) { this.max = max; } + /** Allocate specified number of items */ synchronized int allocate(int n) { final int remaining = max - count; if (remaining <= 0) { @@ -147,9 +150,19 @@ public class Dispatcher { } } + /** Aloocate a single lot of items */ + int allocate() { + return allocate(lotSize); + } + synchronized void reset() { count = 0; } + + /** Set the lot size */ + synchronized void setLotSize(int lotSize) { + this.lotSize = lotSize; + } } private static class GlobalBlockMap { @@ -905,6 +918,7 @@ public class Dispatcher { this.dispatchExecutor = dispatcherThreads == 0? null : Executors.newFixedThreadPool(dispatcherThreads); this.moverThreadAllocator = new Allocator(moverThreads); + this.maxMoverThreads = moverThreads; this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode; this.getBlocksSize = getBlocksSize; @@ -999,7 +1013,7 @@ public class Dispatcher { final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { - final int nThreads = moverThreadAllocator.allocate(maxConcurrentMovesPerNode); + final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } @@ -1050,6 +1064,25 @@ public class Dispatcher { LOG.debug("Disperse Interval sec = " + concurrentThreads / BALANCER_NUM_RPC_PER_SEC); } + + // Determine the size of each mover thread pool per target + int threadsPerTarget = maxMoverThreads/targets.size(); + if (threadsPerTarget == 0) { + // Some scheduled moves will get ignored as some targets won't have + // any threads allocated. + moverThreadAllocator.setLotSize(1); + LOG.warn(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY + "=" + + maxMoverThreads + " is too small for moving blocks to " + + targets.size() + " targets. Balancing may be slower."); + } else { + if (threadsPerTarget > maxConcurrentMovesPerNode) { + threadsPerTarget = maxConcurrentMovesPerNode; + LOG.info("Limiting threads per target to the specified max."); + } + moverThreadAllocator.setLotSize(threadsPerTarget); + LOG.info("Allocating " + threadsPerTarget + " threads per target."); + } + long dSec = 0; final Iterator<Source> i = sources.iterator(); for (int j = 0; j < futures.length; j++) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org