Repository: hbase Updated Branches: refs/heads/branch-1.1 f2b059ea7 -> 4c39c74ff
HBASE-14536 Balancer & SSH interfering with each other leading to unavailability (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c39c74f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c39c74f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c39c74f Branch: refs/heads/branch-1.1 Commit: 4c39c74ff319c94517c53eb1a639ef7ed53d799c Parents: f2b059e Author: Stephen Yuan Jiang <syuanjiang...@gmail.com> Authored: Fri Oct 16 13:27:24 2015 -0700 Committer: Stephen Yuan Jiang <syuanjiang...@gmail.com> Committed: Fri Oct 16 13:28:32 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/master/AssignmentManager.java | 44 ++++++++++++++------ .../hadoop/hbase/master/ServerManager.java | 3 +- .../handler/MetaServerShutdownHandler.java | 32 +++++++++----- .../master/handler/ServerShutdownHandler.java | 3 +- 4 files changed, 57 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4c39c74f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 09e2b2f..bf93ac3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -260,6 +260,10 @@ public class AssignmentManager extends ZooKeeperListener { private RegionStateListener regionStateListener; + public enum ServerHostRegion { + NOT_HOSTING_REGION, HOSTING_REGION, UNKNOWN, + } + /** * Constructs a new assignment manager. * @@ -3382,16 +3386,16 @@ public class AssignmentManager extends ZooKeeperListener { threadPoolExecutorService.submit(new UnAssignCallable(this, regionInfo)); } - public boolean isCarryingMeta(ServerName serverName) { + public ServerHostRegion isCarryingMeta(ServerName serverName) { return isCarryingRegion(serverName, HRegionInfo.FIRST_META_REGIONINFO); } - public boolean isCarryingMetaReplica(ServerName serverName, int replicaId) { + public ServerHostRegion isCarryingMetaReplica(ServerName serverName, int replicaId) { return isCarryingRegion(serverName, RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId)); } - public boolean isCarryingMetaReplica(ServerName serverName, HRegionInfo metaHri) { + public ServerHostRegion isCarryingMetaReplica(ServerName serverName, HRegionInfo metaHri) { return isCarryingRegion(serverName, metaHri); } @@ -3405,7 +3409,7 @@ public class AssignmentManager extends ZooKeeperListener { * processing hasn't finished yet when server shutdown occurs. * @return whether the serverName currently hosts the region */ - private boolean isCarryingRegion(ServerName serverName, HRegionInfo hri) { + private ServerHostRegion isCarryingRegion(ServerName serverName, HRegionInfo hri) { RegionTransition rt = null; try { byte [] data = ZKAssign.getData(watcher, hri.getEncodedName()); @@ -3423,17 +3427,33 @@ public class AssignmentManager extends ZooKeeperListener { boolean matchZK = addressFromZK.equals(serverName); LOG.debug("Checking region=" + hri.getRegionNameAsString() + ", zk server=" + addressFromZK + " current=" + serverName + ", matches=" + matchZK); - return matchZK; + return matchZK ? ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION; } ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri); - boolean matchAM = (addressFromAM != null && - addressFromAM.equals(serverName)); - LOG.debug("based on AM, current region=" + hri.getRegionNameAsString() + - " is on server=" + (addressFromAM != null ? addressFromAM : "null") + - " server being checked: " + serverName); - - return matchAM; + if (addressFromAM != null) { + boolean matchAM = addressFromAM.equals(serverName); + LOG.debug("based on AM, current region=" + hri.getRegionNameAsString() + + " is on server=" + (addressFromAM != null ? addressFromAM : "null") + + " server being checked: " + serverName); + return matchAM ? ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION; + } + + if (hri.isMetaRegion() && RegionReplicaUtil.isDefaultReplica(hri)) { + // For the Meta region (default replica), we can do one more check on MetaTableLocator + final ServerName serverNameInZK = + server.getMetaTableLocator().getMetaRegionLocation(this.server.getZooKeeper()); + LOG.debug("Based on MetaTableLocator, the META region is on server=" + + (serverNameInZK == null ? "null" : serverNameInZK) + + " server being checked: " + serverName); + if (serverNameInZK != null) { + return serverNameInZK.equals(serverName) ? + ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION; + } + } + // Checked everywhere, if reaching here, we are sure that the server is not + // carrying region. + return ServerHostRegion.UNKNOWN; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/4c39c74f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 564a99b..a4d3c6e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -614,7 +614,8 @@ public class ServerManager { return; } - boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName); + boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName) == + AssignmentManager.ServerHostRegion.HOSTING_REGION; if (carryingMeta) { this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master, this.services, this.deadservers, serverName)); http://git-wip-us.apache.org/repos/asf/hbase/blob/4c39c74f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java index f4bc8c3..abf4b20 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java @@ -87,17 +87,27 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { // Assign meta if we were carrying it. // Check again: region may be assigned to other where because of RIT // timeout - if (am.isCarryingMeta(serverName)) { - LOG.info("Server " + serverName + " was carrying META. Trying to assign."); - am.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); - verifyAndAssignMetaWithRetries(); - } else if (!server.getMetaTableLocator().isLocationAvailable(this.server.getZooKeeper())) { - // the meta location as per master is null. This could happen in case when meta assignment - // in previous run failed, while meta znode has been updated to null. We should try to - // assign the meta again. - verifyAndAssignMetaWithRetries(); - } else { - LOG.info("META has been assigned to otherwhere, skip assigning."); + AssignmentManager.ServerHostRegion rsCarryingMetaRegion = am.isCarryingMeta(serverName); + switch (rsCarryingMetaRegion) { + case HOSTING_REGION: + LOG.info("Server " + serverName + " was carrying META. Trying to assign."); + am.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); + verifyAndAssignMetaWithRetries(); + break; + case UNKNOWN: + if (!server.getMetaTableLocator().isLocationAvailable(this.server.getZooKeeper())) { + // the meta location as per master is null. This could happen in case when meta + // assignment in previous run failed, while meta znode has been updated to null. + // We should try to assign the meta again. + verifyAndAssignMetaWithRetries(); + break; + } + // fall through + case NOT_HOSTING_REGION: + LOG.info("META has been assigned to otherwhere, skip assigning."); + break; + default: + throw new IOException("Unsupported action in MetaServerShutdownHandler"); } try { http://git-wip-us.apache.org/repos/asf/hbase/blob/4c39c74f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index 7789ee1..163c942 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -225,7 +225,8 @@ public class ServerShutdownHandler extends EventHandler { for (int i = 1; i < replicaCount; i++) { HRegionInfo metaHri = RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, i); - if (am.isCarryingMetaReplica(serverName, metaHri)) { + if (am.isCarryingMetaReplica(serverName, metaHri) == + AssignmentManager.ServerHostRegion.HOSTING_REGION) { LOG.info("Reassigning meta replica" + metaHri + " that was on " + serverName); toAssignRegions.add(metaHri); }