Two SPLIT requests came in on top of each other; the second failed because it saw parent region was SPLIT. I 'fixed' this before but my fix was in the wrong place
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c613f613 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c613f613 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c613f613 Branch: refs/heads/HBASE-14614 Commit: c613f613b442a1fe155adfd7bd998d79987d6a06 Parents: af2abbd Author: Michael Stack <st...@apache.org> Authored: Mon May 15 09:49:30 2017 -0700 Committer: Michael Stack <st...@apache.org> Committed: Tue May 23 00:33:03 2017 -0700 ---------------------------------------------------------------------- .../hbase/procedure2/ProcedureExecutor.java | 2 +- .../assignment/SplitTableRegionProcedure.java | 20 ++++++++++++-------- .../hadoop/hbase/regionserver/HRegion.java | 5 +++++ 3 files changed, 18 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c613f613/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 258e268..bc73453 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -1147,7 +1147,7 @@ public class ProcedureExecutor<TEnvironment> { scheduler.yield(proc); break; case LOCK_EVENT_WAIT: - // someone will wake us up when the lock is available + // Someone will wake us up when the lock is available LOG.debug(lockState + " " + proc); break; default: http://git-wip-us.apache.org/repos/asf/hbase/blob/c613f613/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 6815e9f..7ebe769 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -341,6 +341,15 @@ public class SplitTableRegionProcedure if (node != null) { parentHRI = node.getRegionInfo(); + // Lookup the parent HRI state from the AM, which has the latest updated info. + // Protect against the case where concurrent SPLIT requests came in. Check a SPLIT + // did not just run. + if (parentHRI.isSplit() || parentHRI.isOffline()) { + setFailure(new IOException("Split " + parentHRI.getRegionNameAsString() + " FAILED because " + + "offline/split already.")); + return false; + } + // expected parent to be online or closed if (!node.isInState(EXPECTED_SPLIT_STATES)) { // We may have SPLIT already? @@ -350,13 +359,6 @@ public class SplitTableRegionProcedure return false; } - // lookup the parent HRI state from the AM, which has the latest updated info. - if (parentHRI.isSplit() || parentHRI.isOffline()) { - setFailure(new IOException("Split " + parentHRI.getRegionNameAsString() + " FAILED because " + - "offline/split already.")); - return false; - } - // Ask the remote regionserver if this region is splittable. If we get an IOE, report it // along w/ the failure so can see why we are not splittable at this time. IOException splittableCheckIOE = null; @@ -365,7 +367,9 @@ public class SplitTableRegionProcedure GetRegionInfoResponse response = Util.getRegionInfoResponse(env, node.getRegionLocation(), node.getRegionInfo()); splittable = response.hasSplittable() && response.getSplittable(); - LOG.info("REMOVE splittable " + splittable + " " + this + " " + node); + if (LOG.isDebugEnabled()) { + LOG.debug("Splittable=" + splittable + " " + this + " " + node.toShortString()); + } } catch (IOException e) { splittableCheckIOE = e; } http://git-wip-us.apache.org/repos/asf/hbase/blob/c613f613/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index ac0867c..8cc9cd7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7800,6 +7800,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return null; } + // Can't split a region that is closing. + if (this.isClosing()) { + return null; + } + if (!splitPolicy.shouldSplit()) { return null; }