HBASE-19815 Flakey TestAssignmentManager.testAssignWithRandExec (Part Two).
Part One cleaned up a ClassCastException. Part Two adds the ServerCrashProcedure#handleRIT behavior to RecoverMetaProcedure. Adds debug in the test. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/581fabe7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/581fabe7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/581fabe7 Branch: refs/heads/HBASE-19064 Commit: 581fabe7b2177a090af33517f2f7cb1cdab2c64b Parents: 646770d Author: Michael Stack <st...@apache.org> Authored: Wed Jan 17 22:35:35 2018 -0800 Committer: Michael Stack <st...@apache.org> Committed: Thu Jan 18 11:32:21 2018 -0800 ---------------------------------------------------------------------- .../master/procedure/RecoverMetaProcedure.java | 38 +++++++++++++++++--- .../assignment/TestAssignmentManager.java | 9 ++++- 2 files changed, 41 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/581fabe7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java index 50ef3e0..70d0d55 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java @@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.assignment.AssignProcedure; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; @@ -126,17 +128,17 @@ public class RecoverMetaProcedure RegionInfoBuilder.FIRST_META_REGIONINFO, this.replicaId); AssignProcedure metaAssignProcedure; + AssignmentManager am = master.getAssignmentManager(); if (failedMetaServer != null) { - LOG.info(this + "; Assigning meta with new plan. previous meta server=" + - failedMetaServer); - metaAssignProcedure = master.getAssignmentManager().createAssignProcedure(hri); + handleRIT(env, hri, this.failedMetaServer); + LOG.info(this + "; Assigning meta with new plan; previous server=" + failedMetaServer); + metaAssignProcedure = am.createAssignProcedure(hri); } else { // get server carrying meta from zk ServerName metaServer = MetaTableLocator.getMetaRegionState(master.getZooKeeper()).getServerName(); LOG.info(this + "; Retaining meta assignment to server=" + metaServer); - metaAssignProcedure = - master.getAssignmentManager().createAssignProcedure(hri, metaServer); + metaAssignProcedure = am.createAssignProcedure(hri, metaServer); } addChildProcedure(metaAssignProcedure); @@ -152,6 +154,32 @@ public class RecoverMetaProcedure return Flow.HAS_MORE_STATE; } + /** + * Is the region stuck assigning to this failedMetaServer? If so, cancel the call + * just as we do over in ServerCrashProcedure#handleRIT except less to do here; less context + * to carry. + */ + private void handleRIT(MasterProcedureEnv env, RegionInfo ri, ServerName crashedServerName) { + AssignmentManager am = env.getAssignmentManager(); + RegionTransitionProcedure rtp = am.getRegionStates().getRegionTransitionProcedure(ri); + if (rtp == null) { + return; // Nothing to do. Not in RIT. + } + // Make sure the RIT is against this crashed server. In the case where there are many + // processings of a crashed server -- backed up for whatever reason (slow WAL split) + // -- then a previous SCP may have already failed an assign, etc., and it may have a + // new location target; DO NOT fail these else we make for assign flux. + ServerName rtpServerName = rtp.getServer(env); + if (rtpServerName == null) { + LOG.warn("RIT with ServerName null! " + rtp); + } else if (rtpServerName.equals(crashedServerName)) { + LOG.info("pid=" + getProcId() + " found RIT " + rtp + "; " + + rtp.getRegionState(env).toShortString()); + rtp.remoteCallFailed(env, crashedServerName, + new ServerCrashException(getProcId(), crashedServerName)); + } + } + @Override protected void rollbackState(MasterProcedureEnv env, MasterProcedureProtos.RecoverMetaState recoverMetaState) http://git-wip-us.apache.org/repos/asf/hbase/blob/581fabe7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java index 3ab915b..9b9f624 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.SocketTimeoutException; import java.util.NavigableMap; import java.util.Random; @@ -206,7 +207,7 @@ public class TestAssignmentManager { rsDispatcher.setMockRsExecutor(new RandRsExecutor()); // Loop a bunch of times so we hit various combos of exceptions. for (int i = 0; i < 10; i++) { - LOG.info("" + i); + LOG.info("ROUND=" + i); AssignProcedure proc = am.createAssignProcedure(hri); waitOnFuture(submitProcedure(proc)); } @@ -445,6 +446,12 @@ public class TestAssignmentManager { return future.get(5, TimeUnit.SECONDS); } catch (ExecutionException e) { LOG.info("ExecutionException", e); + Exception ee = (Exception)e.getCause(); + if (ee instanceof InterruptedIOException) { + for (Procedure p: this.master.getMasterProcedureExecutor().getProcedures()) { + LOG.info(p.toStringDetails()); + } + } throw (Exception)e.getCause(); } }
