HBASE-20548 Master fails to startup on large clusters, refreshing block distribution
Signed-off-by: Andrew Purtell <apurt...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b892be74 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b892be74 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b892be74 Branch: refs/heads/branch-1 Commit: b892be7440afde8da8a2db8fe3053af3ba4daedf Parents: 416f283 Author: Thiruvel Thirumoolan <thiru...@oath.com> Authored: Tue May 22 18:28:31 2018 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Thu May 24 12:27:19 2018 -0700 ---------------------------------------------------------------------- .../hbase/rsgroup/RSGroupBasedLoadBalancer.java | 5 ++++ .../org/apache/hadoop/hbase/master/HMaster.java | 11 +++++++++ .../hadoop/hbase/master/LoadBalancer.java | 5 ++++ .../hbase/master/balancer/BaseLoadBalancer.java | 26 +++++++++++++------- 4 files changed, 38 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index ebae030..77adf0d 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -455,4 +455,9 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc public boolean isStopped() { return false; } + + @Override + public void postMasterStartupInitialize() { + this.internalBalancer.postMasterStartupInitialize(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 85d5e23..a2b43e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -933,6 +933,17 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } zombieDetector.interrupt(); + + /* + * After master has started up, lets do balancer post startup initialization. Since this runs + * in activeMasterManager thread, it should be fine. + */ + long start = System.currentTimeMillis(); + this.balancer.postMasterStartupInitialize(); + if (LOG.isDebugEnabled()) { + LOG.debug("Balancer post startup initialization complete, took " + ( + (System.currentTimeMillis() - start) / 1000) + " seconds"); + } } private void initQuotaManager() throws IOException { http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java index e387f59..8531ff7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java @@ -152,4 +152,9 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse */ @Override void onConfigurationChange(Configuration conf); + + /** + * If balancer needs to do initialization after Master has started up, lets do that here. + */ + void postMasterStartupInitialize(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 41cbeaa..01df8a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -1267,6 +1267,19 @@ public abstract class BaseLoadBalancer implements LoadBalancer { } } + @Override + public void postMasterStartupInitialize() { + if (services != null && regionFinder != null) { + try { + Set<HRegionInfo> regions = + services.getAssignmentManager().getRegionStates().getRegionAssignments().keySet(); + regionFinder.refreshAndWait(regions); + } catch (Exception e) { + LOG.warn("Refreshing region HDFS Block dist failed with exception, ignoring", e); + } + } + } + public void setRackManager(RackManager rackManager) { this.rackManager = rackManager; } @@ -1367,7 +1380,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { return assignments; } - Cluster cluster = createCluster(servers, regions, false); + Cluster cluster = createCluster(servers, regions); List<HRegionInfo> unassignedRegions = new ArrayList<HRegionInfo>(); roundRobinAssignment(cluster, regions, unassignedRegions, @@ -1413,11 +1426,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { return assignments; } - protected Cluster createCluster(List<ServerName> servers, - Collection<HRegionInfo> regions, boolean forceRefresh) { - if (forceRefresh && useRegionFinder) { - regionFinder.refreshAndWait(regions); - } + protected Cluster createCluster(List<ServerName> servers, Collection<HRegionInfo> regions) { // Get the snapshot of the current assignments for the regions in question, and then create // a cluster out of it. Note that we might have replicas already assigned to some servers // earlier. So we want to get the snapshot to see those assignments, but this will only contain @@ -1491,7 +1500,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { } List<HRegionInfo> regions = Lists.newArrayList(regionInfo); - Cluster cluster = createCluster(servers, regions, false); + Cluster cluster = createCluster(servers, regions); return randomAssignment(cluster, regionInfo, servers); } @@ -1569,8 +1578,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer { int numRandomAssignments = 0; int numRetainedAssigments = 0; - Cluster cluster = createCluster(servers, regions.keySet(), true); - for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) { HRegionInfo region = entry.getKey(); ServerName oldServerName = entry.getValue(); @@ -1613,6 +1620,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { // If servers from prior assignment aren't present, then lets do randomAssignment on regions. if (randomAssignRegions.size() > 0) { + Cluster cluster = createCluster(servers, regions.keySet()); for (Map.Entry<ServerName, List<HRegionInfo>> entry : assignments.entrySet()) { ServerName sn = entry.getKey(); for (HRegionInfo region : entry.getValue()) {
