HBASE-14604 Improve MoveCostFunction in StochasticLoadBalancer (Guanghao Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/60465964 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/60465964 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/60465964 Branch: refs/heads/hbase-12439 Commit: 60465964039acd05f43f268cdb4f909a150a0f41 Parents: c9523a5 Author: tedyu <yuzhih...@gmail.com> Authored: Tue Oct 20 02:35:40 2015 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Tue Oct 20 02:35:40 2015 -0700 ---------------------------------------------------------------------- .../hbase/master/balancer/BaseLoadBalancer.java | 11 ++++++ .../master/balancer/StochasticLoadBalancer.java | 2 +- .../balancer/TestStochasticLoadBalancer.java | 38 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/60465964/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 962b241..33e8d97 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.T import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; @@ -898,6 +899,16 @@ public abstract class BaseLoadBalancer implements LoadBalancer { } } + @VisibleForTesting + protected void setNumRegions(int numRegions) { + this.numRegions = numRegions; + } + + @VisibleForTesting + protected void setNumMovedRegions(int numMovedRegions) { + this.numMovedRegions = numMovedRegions; + } + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SBSC_USE_STRINGBUFFER_CONCATENATION", justification="Not important but should be fixed") @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/60465964/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 249423f..f9b3baf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -1026,7 +1026,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { return 1000000; // return a number much greater than any of the other cost } - return scale(0, cluster.numRegions, moveCost); + return scale(0, Math.min(cluster.numRegions, maxMoves), moveCost); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/60465964/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index 85ecb29..7abbeb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -122,6 +122,44 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { } @Test + public void testMoveCost() throws Exception { + Configuration conf = HBaseConfiguration.create(); + StochasticLoadBalancer.CostFunction + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + for (int[] mockCluster : clusterStateMocks) { + BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster); + costFunction.init(cluster); + double cost = costFunction.cost(); + assertEquals(0.0f, cost, 0.001); + + // cluster region number is smaller than maxMoves=600 + cluster.setNumRegions(200); + cluster.setNumMovedRegions(10); + cost = costFunction.cost(); + assertEquals(0.05f, cost, 0.001); + cluster.setNumMovedRegions(100); + cost = costFunction.cost(); + assertEquals(0.5f, cost, 0.001); + cluster.setNumMovedRegions(200); + cost = costFunction.cost(); + assertEquals(1.0f, cost, 0.001); + + + // cluster region number is bigger than maxMoves=2500 + cluster.setNumRegions(10000); + cluster.setNumMovedRegions(250); + cost = costFunction.cost(); + assertEquals(0.1f, cost, 0.001); + cluster.setNumMovedRegions(1250); + cost = costFunction.cost(); + assertEquals(0.5f, cost, 0.001); + cluster.setNumMovedRegions(2500); + cost = costFunction.cost(); + assertEquals(1.0f, cost, 0.01); + } + } + + @Test public void testSkewCost() { Configuration conf = HBaseConfiguration.create(); StochasticLoadBalancer.CostFunction