Repository: hbase Updated Branches: refs/heads/master 28d8609e5 -> f04eeecff
http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DispatchMergingRegionsProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DispatchMergingRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DispatchMergingRegionsProcedure.java new file mode 100644 index 0000000..0dfb711 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DispatchMergingRegionsProcedure.java @@ -0,0 +1,584 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.RegionLoad; +import org.apache.hadoop.hbase.ServerLoad; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.exceptions.MergeRegionException; +import org.apache.hadoop.hbase.exceptions.RegionOpeningException; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.CatalogJanitor; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DispatchMergingRegionsState; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * The procedure to Merge a region in a table. + */ +@InterfaceAudience.Private +public class DispatchMergingRegionsProcedure +extends StateMachineProcedure<MasterProcedureEnv, DispatchMergingRegionsState> +implements TableProcedureInterface { + private static final Log LOG = LogFactory.getLog(DispatchMergingRegionsProcedure.class); + + private final AtomicBoolean aborted = new AtomicBoolean(false); + private Boolean traceEnabled; + private AssignmentManager assignmentManager; + private int timeout; + private ServerName regionLocation; + private String regionsToMergeListFullName; + private String regionsToMergeListEncodedName; + + private UserGroupInformation user; + private TableName tableName; + private HRegionInfo [] regionsToMerge; + private boolean forcible; + + public DispatchMergingRegionsProcedure() { + this.traceEnabled = isTraceEnabled(); + this.assignmentManager = null; + this.timeout = -1; + this.regionLocation = null; + this.regionsToMergeListFullName = null; + this.regionsToMergeListEncodedName = null; + } + + public DispatchMergingRegionsProcedure( + final MasterProcedureEnv env, + final TableName tableName, + final HRegionInfo [] regionsToMerge, + final boolean forcible) throws IOException { + this.traceEnabled = isTraceEnabled(); + this.assignmentManager = getAssignmentManager(env); + this.tableName = tableName; + // For now, we only merge 2 regions. It could be extended to more than 2 regions in + // the future. + assert(regionsToMerge.length == 2); + this.regionsToMerge = regionsToMerge; + this.forcible = forcible; + + this.user = env.getRequestUser().getUGI(); + this.setOwner(this.user.getShortUserName()); + + this.timeout = -1; + this.regionsToMergeListFullName = getRegionsToMergeListFullNameString(); + this.regionsToMergeListEncodedName = getRegionsToMergeListEncodedNameString(); + } + + @Override + protected Flow executeFromState( + final MasterProcedureEnv env, + final DispatchMergingRegionsState state) throws InterruptedException { + if (isTraceEnabled()) { + LOG.trace(this + " execute state=" + state); + } + + try { + switch (state) { + case DISPATCH_MERGING_REGIONS_PREPARE: + prepareMergeRegion(env); + setNextState(DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_PRE_OPERATION); + break; + case DISPATCH_MERGING_REGIONS_PRE_OPERATION: + //Unused for now - reserve to add preMerge coprocessor in the future + setNextState(DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_MOVE_REGION_TO_SAME_RS); + break; + case DISPATCH_MERGING_REGIONS_MOVE_REGION_TO_SAME_RS: + if (MoveRegionsToSameRS(env)) { + setNextState(DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_DO_MERGE_IN_RS); + } else { + LOG.info("Cancel merging regions " + getRegionsToMergeListFullNameString() + + ", because can't move them to the same RS"); + setNextState(DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_POST_OPERATION); + } + break; + case DISPATCH_MERGING_REGIONS_DO_MERGE_IN_RS: + doMergeInRS(env); + setNextState(DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_POST_OPERATION); + break; + case DISPATCH_MERGING_REGIONS_POST_OPERATION: + //Unused for now - reserve to add postCompletedMerge coprocessor in the future + return Flow.NO_MORE_STATE; + default: + throw new UnsupportedOperationException(this + " unhandled state=" + state); + } + } catch (IOException e) { + LOG.warn("Error trying to merge regions " + getRegionsToMergeListFullNameString() + + " in the table " + tableName + " (in state=" + state + ")", e); + + setFailure("master-merge-regions", e); + } + return Flow.HAS_MORE_STATE; + } + + @Override + protected void rollbackState( + final MasterProcedureEnv env, + final DispatchMergingRegionsState state) throws IOException, InterruptedException { + if (isTraceEnabled()) { + LOG.trace(this + " rollback state=" + state); + } + + try { + switch (state) { + case DISPATCH_MERGING_REGIONS_POST_OPERATION: + break; // nothing to rollback + case DISPATCH_MERGING_REGIONS_DO_MERGE_IN_RS: + String msg = this + " We are in the " + state + " state." + + " It is complicated to rollback the merge operation that region server is working on." + + " Rollback is not supported and we should let the merge operation to complete"; + LOG.warn(msg); + break; + case DISPATCH_MERGING_REGIONS_MOVE_REGION_TO_SAME_RS: + break; // nothing to rollback + case DISPATCH_MERGING_REGIONS_PRE_OPERATION: + break; // nothing to rollback + case DISPATCH_MERGING_REGIONS_PREPARE: + break; // nothing to rollback + default: + throw new UnsupportedOperationException(this + " unhandled state=" + state); + } + } catch (Exception e) { + // This will be retried. Unless there is a bug in the code, + // this should be just a "temporary error" (e.g. network down) + LOG.warn("Failed rollback attempt step " + state + " for merging the regions " + + getRegionsToMergeListFullNameString() + " in table " + tableName, e); + throw e; + } + } + + @Override + protected DispatchMergingRegionsState getState(final int stateId) { + return DispatchMergingRegionsState.valueOf(stateId); + } + + @Override + protected int getStateId(final DispatchMergingRegionsState state) { + return state.getNumber(); + } + + @Override + protected DispatchMergingRegionsState getInitialState() { + return DispatchMergingRegionsState.DISPATCH_MERGING_REGIONS_PREPARE; + } + + @Override + protected void setNextState(DispatchMergingRegionsState state) { + if (aborted.get()) { + setAbortFailure("merge-table-regions", "abort requested"); + } else { + super.setNextState(state); + } + } + + @Override + public boolean abort(final MasterProcedureEnv env) { + aborted.set(true); + return true; + } + + @Override + public void serializeStateData(final OutputStream stream) throws IOException { + super.serializeStateData(stream); + + MasterProcedureProtos.DispatchMergingRegionsStateData.Builder dispatchMergingRegionsMsg = + MasterProcedureProtos.DispatchMergingRegionsStateData.newBuilder() + .setUserInfo(MasterProcedureUtil.toProtoUserInfo(user)) + .setTableName(ProtobufUtil.toProtoTableName(tableName)) + .setForcible(forcible); + for (HRegionInfo hri: regionsToMerge) { + dispatchMergingRegionsMsg.addRegionInfo(HRegionInfo.convert(hri)); + } + dispatchMergingRegionsMsg.build().writeDelimitedTo(stream); + } + + @Override + public void deserializeStateData(final InputStream stream) throws IOException { + super.deserializeStateData(stream); + + MasterProcedureProtos.DispatchMergingRegionsStateData dispatchMergingRegionsMsg = + MasterProcedureProtos.DispatchMergingRegionsStateData.parseDelimitedFrom(stream); + user = MasterProcedureUtil.toUserInfo(dispatchMergingRegionsMsg.getUserInfo()); + tableName = ProtobufUtil.toTableName(dispatchMergingRegionsMsg.getTableName()); + + assert(dispatchMergingRegionsMsg.getRegionInfoCount() == 2); + regionsToMerge = new HRegionInfo[dispatchMergingRegionsMsg.getRegionInfoCount()]; + for (int i = 0; i < regionsToMerge.length; i++) { + regionsToMerge[i] = HRegionInfo.convert(dispatchMergingRegionsMsg.getRegionInfo(i)); + } + } + + @Override + public void toStringClassDetails(StringBuilder sb) { + sb.append(getClass().getSimpleName()); + sb.append(" (table="); + sb.append(tableName); + sb.append(" regions="); + sb.append(getRegionsToMergeListFullNameString()); + sb.append(" forcible="); + sb.append(forcible); + sb.append(")"); + } + + @Override + protected boolean acquireLock(final MasterProcedureEnv env) { + return env.getProcedureQueue().waitRegions( + this, getTableName(), regionsToMerge[0], regionsToMerge[1]); + } + + @Override + protected void releaseLock(final MasterProcedureEnv env) { + env.getProcedureQueue().wakeRegions(this, getTableName(), regionsToMerge[0], regionsToMerge[1]); + } + + @Override + public TableName getTableName() { + return tableName; + } + + @Override + public TableOperationType getTableOperationType() { + return TableOperationType.MERGE; + } + + /** + * Prepare merge and do some check + * @param env MasterProcedureEnv + * @throws IOException + */ + private void prepareMergeRegion(final MasterProcedureEnv env) throws IOException { + // Note: the following logic assumes that we only have 2 regions to merge. In the future, + // if we want to extend to more than 2 regions, the code needs to modify a little bit. + // + CatalogJanitor catalogJanitor = env.getMasterServices().getCatalogJanitor(); + boolean regionAHasMergeQualifier = !catalogJanitor.cleanMergeQualifier(regionsToMerge[0]); + if (regionAHasMergeQualifier + || !catalogJanitor.cleanMergeQualifier(regionsToMerge[1])) { + String msg = "Skip merging regions " + regionsToMerge[0].getRegionNameAsString() + + ", " + regionsToMerge[1].getRegionNameAsString() + ", because region " + + (regionAHasMergeQualifier ? regionsToMerge[0].getEncodedName() : regionsToMerge[1] + .getEncodedName()) + " has merge qualifier"; + LOG.info(msg); + throw new MergeRegionException(msg); + } + + RegionStates regionStates = getAssignmentManager(env).getRegionStates(); + RegionState regionStateA = regionStates.getRegionState(regionsToMerge[0].getEncodedName()); + RegionState regionStateB = regionStates.getRegionState(regionsToMerge[1].getEncodedName()); + if (regionStateA == null || regionStateB == null) { + throw new UnknownRegionException( + regionStateA == null ? + regionsToMerge[0].getEncodedName() : regionsToMerge[1].getEncodedName()); + } + + if (!regionStateA.isOpened() || !regionStateB.isOpened()) { + throw new MergeRegionException( + "Unable to merge regions not online " + regionStateA + ", " + regionStateB); + } + + if (regionsToMerge[0].getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID || + regionsToMerge[1].getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { + throw new MergeRegionException("Can't merge non-default replicas"); + } + + if (!forcible && !HRegionInfo.areAdjacent(regionsToMerge[0], regionsToMerge[1])) { + throw new MergeRegionException( + "Unable to merge not adjacent regions " + + regionsToMerge[0].getRegionNameAsString() + ", " + + regionsToMerge[1].getRegionNameAsString() + + " where forcible = " + forcible); + } + } + + /** + * Move all regions to the same region server + * @param env MasterProcedureEnv + * @return whether target regions hosted by the same RS + * @throws IOException + */ + private boolean MoveRegionsToSameRS(final MasterProcedureEnv env) throws IOException { + // Make sure regions are on the same regionserver before send merge + // regions request to region server. + // + boolean onSameRS = isRegionsOnTheSameServer(env); + if (!onSameRS) { + // Note: the following logic assumes that we only have 2 regions to merge. In the future, + // if we want to extend to more than 2 regions, the code needs to modify a little bit. + // + RegionStates regionStates = getAssignmentManager(env).getRegionStates(); + ServerName regionLocation2 = regionStates.getRegionServerOfRegion(regionsToMerge[1]); + + RegionLoad loadOfRegionA = getRegionLoad(env, regionLocation, regionsToMerge[0]); + RegionLoad loadOfRegionB = getRegionLoad(env, regionLocation2, regionsToMerge[1]); + if (loadOfRegionA != null && loadOfRegionB != null + && loadOfRegionA.getRequestsCount() < loadOfRegionB.getRequestsCount()) { + // switch regionsToMerge[0] and regionsToMerge[1] + HRegionInfo tmpRegion = this.regionsToMerge[0]; + this.regionsToMerge[0] = this.regionsToMerge[1]; + this.regionsToMerge[1] = tmpRegion; + ServerName tmpLocation = regionLocation; + regionLocation = regionLocation2; + regionLocation2 = tmpLocation; + } + + long startTime = EnvironmentEdgeManager.currentTime(); + + RegionPlan regionPlan = new RegionPlan(regionsToMerge[1], regionLocation2, regionLocation); + LOG.info("Moving regions to same server for merge: " + regionPlan.toString()); + getAssignmentManager(env).balance(regionPlan); + do { + try { + Thread.sleep(20); + // Make sure check RIT first, then get region location, otherwise + // we would make a wrong result if region is online between getting + // region location and checking RIT + boolean isRIT = regionStates.isRegionInTransition(regionsToMerge[1]); + regionLocation2 = regionStates.getRegionServerOfRegion(regionsToMerge[1]); + onSameRS = regionLocation.equals(regionLocation2); + if (onSameRS || !isRIT) { + // Regions are on the same RS, or regionsToMerge[1] is not in + // RegionInTransition any more + break; + } + } catch (InterruptedException e) { + InterruptedIOException iioe = new InterruptedIOException(); + iioe.initCause(e); + throw iioe; + } + } while ((EnvironmentEdgeManager.currentTime() - startTime) <= getTimeout(env)); + } + return onSameRS; + } + + /** + * Do the real merge operation in the region server that hosts regions + * @param env MasterProcedureEnv + * @throws IOException + */ + private void doMergeInRS(final MasterProcedureEnv env) throws IOException { + long duration = 0; + long startTime = EnvironmentEdgeManager.currentTime(); + do { + try { + if (getServerName(env) == null) { + // The merge probably already happen. Check + RegionState regionState = getAssignmentManager(env).getRegionStates().getRegionState( + regionsToMerge[0].getEncodedName()); + if (regionState.isMerging() || regionState.isMerged()) { + LOG.info("Merge regions " + getRegionsToMergeListEncodedNameString() + + " is in progress or completed. No need to send a new request."); + } else { + LOG.warn("Cannot sending merge to hosting server of the regions " + + getRegionsToMergeListEncodedNameString() + " as the server is unknown"); + } + return; + } + // TODO: the following RPC call is not idempotent. Multiple calls (eg. after master + // failover, re-execute this step) could result in some exception thrown that does not + // paint the correct picture. This behavior is on-par with old releases. Improvement + // could happen in the future. + env.getMasterServices().getServerManager().sendRegionsMerge( + getServerName(env), + regionsToMerge[0], + regionsToMerge[1], + forcible, + user); + LOG.info("Sent merge to server " + getServerName(env) + " for region " + + getRegionsToMergeListEncodedNameString() + ", focible=" + forcible); + return; + } catch (RegionOpeningException roe) { + // Do a retry since region should be online on RS immediately + LOG.warn("Failed mergering regions in " + getServerName(env) + ", retrying...", roe); + } catch (Exception ie) { + LOG.warn("Failed sending merge to " + getServerName(env) + " for regions " + + getRegionsToMergeListEncodedNameString() + ", focible=" + forcible, ie); + return; + } + } while ((duration = EnvironmentEdgeManager.currentTime() - startTime) <= getTimeout(env)); + + // If we reaches here, it means that we get timed out. + String msg = "Failed sending merge to " + getServerName(env) + " after " + duration + "ms"; + LOG.warn(msg); + throw new IOException(msg); + } + + private RegionLoad getRegionLoad( + final MasterProcedureEnv env, + final ServerName sn, + final HRegionInfo hri) { + ServerManager serverManager = env.getMasterServices().getServerManager(); + ServerLoad load = serverManager.getLoad(sn); + if (load != null) { + Map<byte[], RegionLoad> regionsLoad = load.getRegionsLoad(); + if (regionsLoad != null) { + return regionsLoad.get(hri.getRegionName()); + } + } + return null; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @param env MasterProcedureEnv + * @return whether target regions hosted by the same RS + */ + private boolean isRegionsOnTheSameServer(final MasterProcedureEnv env) throws IOException{ + Boolean onSameRS = true; + int i = 0; + RegionStates regionStates = getAssignmentManager(env).getRegionStates(); + regionLocation = regionStates.getRegionServerOfRegion(regionsToMerge[i]); + if (regionLocation != null) { + for(i = 1; i < regionsToMerge.length; i++) { + ServerName regionLocation2 = regionStates.getRegionServerOfRegion(regionsToMerge[i]); + if (regionLocation2 != null) { + if (onSameRS) { + onSameRS = regionLocation.equals(regionLocation2); + } + } else { + // At least one region is not online, merge will fail, no need to continue. + break; + } + } + if (i == regionsToMerge.length) { + // Finish checking all regions, return the result; + return onSameRS; + } + } + + // If reaching here, at least one region is not online. + String msg = "Skip merging regions " + getRegionsToMergeListFullNameString() + + ", because region " + regionsToMerge[i].getEncodedName() + " is not online now."; + LOG.warn(msg); + throw new IOException(msg); + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @param env MasterProcedureEnv + * @return assignmentManager + */ + private AssignmentManager getAssignmentManager(final MasterProcedureEnv env) { + if (assignmentManager == null) { + assignmentManager = env.getMasterServices().getAssignmentManager(); + } + return assignmentManager; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @param env MasterProcedureEnv + * @return timeout value + */ + private int getTimeout(final MasterProcedureEnv env) { + if (timeout == -1) { + timeout = env.getMasterConfiguration().getInt( + "hbase.master.regionmerge.timeout", regionsToMerge.length * 60 * 1000); + } + return timeout; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @param env MasterProcedureEnv + * @return serverName + */ + private ServerName getServerName(final MasterProcedureEnv env) { + if (regionLocation == null) { + regionLocation = + getAssignmentManager(env).getRegionStates().getRegionServerOfRegion(regionsToMerge[0]); + } + return regionLocation; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @param fullName whether return only encoded name + * @return region names in a list + */ + private String getRegionsToMergeListFullNameString() { + if (regionsToMergeListFullName == null) { + StringBuilder sb = new StringBuilder("["); + int i = 0; + while(i < regionsToMerge.length - 1) { + sb.append(regionsToMerge[i].getRegionNameAsString() + ", "); + i++; + } + sb.append(regionsToMerge[i].getRegionNameAsString() + " ]"); + regionsToMergeListFullName = sb.toString(); + } + return regionsToMergeListFullName; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @return encoded region names + */ + private String getRegionsToMergeListEncodedNameString() { + if (regionsToMergeListEncodedName == null) { + StringBuilder sb = new StringBuilder("["); + int i = 0; + while(i < regionsToMerge.length - 1) { + sb.append(regionsToMerge[i].getEncodedName() + ", "); + i++; + } + sb.append(regionsToMerge[i].getEncodedName() + " ]"); + regionsToMergeListEncodedName = sb.toString(); + } + return regionsToMergeListEncodedName; + } + + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @return traceEnabled + */ + private Boolean isTraceEnabled() { + if (traceEnabled == null) { + traceEnabled = LOG.isTraceEnabled(); + } + return traceEnabled; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index fd55f66..ec64eac 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -1180,7 +1180,7 @@ public class TestAdmin1 { gotException = false; // Try merging a replica with another. Should fail. try { - TEST_UTIL.getAdmin().mergeRegions(regions.get(1).getFirst().getEncodedNameAsBytes(), + TEST_UTIL.getAdmin().mergeRegionsAsync(regions.get(1).getFirst().getEncodedNameAsBytes(), regions.get(2).getFirst().getEncodedNameAsBytes(), true); } catch (IllegalArgumentException m) { gotException = true; @@ -1376,7 +1376,7 @@ public class TestAdmin1 { assertEquals(3, admin.getTableRegions(tableName).size()); regionA = tableRegions.get(0); regionB = tableRegions.get(1); - admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false); + admin.mergeRegionsAsync(regionA.getRegionName(), regionB.getRegionName(), false); Thread.sleep(1000); assertEquals(2, admin.getTableRegions(tableName).size()); @@ -1384,7 +1384,8 @@ public class TestAdmin1 { tableRegions = admin.getTableRegions(tableName); regionA = tableRegions.get(0); regionB = tableRegions.get(1); - admin.mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); + admin.mergeRegionsAsync( + regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); Thread.sleep(1000); assertEquals(1, admin.getTableRegions(tableName).size()); } finally { http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java index 45093bb..3948d18 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest; @@ -272,19 +271,6 @@ public class TestHBaseAdminNoCluster { (IsCatalogJanitorEnabledRequest)Mockito.any()); } }); - // Admin.mergeRegions() - testMasterOperationIsRetried(new MethodCaller() { - @Override - public void call(Admin admin) throws Exception { - admin.mergeRegions(new byte[0], new byte[0], true); - } - @Override - public void verify(MasterKeepAliveConnection masterAdmin, int count) throws Exception { - Mockito.verify(masterAdmin, Mockito.atLeast(count)) - .dispatchMergingRegions((RpcController)Mockito.any(), - (DispatchMergingRegionsRequest)Mockito.any()); - } - }); } private static interface MethodCaller { http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java index c5ca0b5..e10cde5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java @@ -116,7 +116,7 @@ public class TestSplitOrMergeStatus { assertTrue(results[0]); List<HRegionInfo> regions = admin.getTableRegions(t.getName()); assertTrue(regions.size() > 1); - admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(), regions.get(1).getEncodedNameAsBytes(), true); int count = waitOnSplitOrMerge(t).size(); assertTrue(orignalCount == count); @@ -125,7 +125,7 @@ public class TestSplitOrMergeStatus { results = admin.setSplitOrMergeEnabled(true, false, true, MasterSwitchType.MERGE); assertEquals(results.length, 1); assertFalse(results[0]); - admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(), regions.get(1).getEncodedNameAsBytes(), true); count = waitOnSplitOrMerge(t).size(); assertTrue(orignalCount>count); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index b4e93bd..7a7d448 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -1541,7 +1541,7 @@ public class TestMasterObserver { RegionLocator regionLocator = connection.getRegionLocator(htd.getTableName()); List<HRegionLocation> regions = regionLocator.getAllRegionLocations(); - admin.mergeRegions(regions.get(0).getRegionInfo().getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(0).getRegionInfo().getEncodedNameAsBytes(), regions.get(1).getRegionInfo().getEncodedNameAsBytes(), true); assertTrue("Coprocessor should have been called on region merge", cp.wasDispatchMergeCalled()); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerObserver.java index 9fcfd43..ecf9da1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerObserver.java @@ -92,8 +92,10 @@ public class TestRegionServerObserver { admin.createTable(desc, new byte[][] { Bytes.toBytes("row") }); assertFalse(regionServerObserver.wasRegionMergeCalled()); List<Region> regions = regionServer.getOnlineRegions(TableName.valueOf(TABLENAME)); - admin.mergeRegions(regions.get(0).getRegionInfo().getEncodedNameAsBytes(), regions.get(1) - .getRegionInfo().getEncodedNameAsBytes(), true); + admin.mergeRegionsAsync( + regions.get(0).getRegionInfo().getEncodedNameAsBytes(), + regions.get(1).getRegionInfo().getEncodedNameAsBytes(), + true); int regionsCount = regionServer.getOnlineRegions(TableName.valueOf(TABLENAME)).size(); while (regionsCount != 1) { regionsCount = regionServer.getOnlineRegions(TableName.valueOf(TABLENAME)).size(); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index 60b62e4..c7a42d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -93,6 +93,11 @@ public class MockNoopMasterServices implements MasterServices, Server { } @Override + public CatalogJanitor getCatalogJanitor() { + return null; + } + + @Override public MasterFileSystem getMasterFileSystem() { return null; } @@ -282,8 +287,13 @@ public class MockNoopMasterServices implements MasterServices, Server { } @Override - public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b, - boolean forcible, User user) throws IOException { + public long dispatchMergingRegions( + final HRegionInfo region_a, + final HRegionInfo region_b, + final boolean forcible, + final long nonceGroup, + final long nonce) throws IOException { + return -1; } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java index 5b3abea..6ad2c5d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java @@ -242,7 +242,7 @@ public class TestAssignmentListener { listener.reset(); List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME); assertEquals(2, regions.size()); - admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(), regions.get(1).getEncodedNameAsBytes(), true); listener.awaitModifications(3); assertEquals(1, admin.getTableRegions(TABLE_NAME).size()); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java new file mode 100644 index 0000000..17d1e757c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java @@ -0,0 +1,296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.procedure; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DispatchMergingRegionsState; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({MasterTests.class, MediumTests.class}) +public class TestDispatchMergingRegionsProcedure { + private static final Log LOG = LogFactory.getLog(TestDispatchMergingRegionsProcedure.class); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static long nonceGroup = HConstants.NO_NONCE; + private static long nonce = HConstants.NO_NONCE; + + private final static byte[] FAMILY = Bytes.toBytes("FAMILY"); + final static Configuration conf = UTIL.getConfiguration(); + private static Admin admin; + + private static void setupConf(Configuration conf) { + // Reduce the maximum attempts to speed up the test + conf.setInt("hbase.assignment.maximum.attempts", 3); + conf.setInt("hbase.master.maximum.ping.server.attempts", 3); + conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1); + + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 3); + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(conf); + UTIL.startMiniCluster(1); + admin = UTIL.getHBaseAdmin(); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + resetProcExecutorTestingKillFlag(); + nonceGroup = + MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster()); + nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster()); + // Turn off balancer so it doesn't cut in and mess up our placements. + UTIL.getHBaseAdmin().setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + resetProcExecutorTestingKillFlag(); + } + + @After + public void tearDown() throws Exception { + resetProcExecutorTestingKillFlag(); + for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) { + LOG.info("Tear down, remove table=" + htd.getTableName()); + UTIL.deleteTable(htd.getTableName()); + } + } + + private void resetProcExecutorTestingKillFlag() { + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); + assertTrue("expected executor to be running", procExec.isRunning()); + } + + /** + * This tests two region merges + */ + @Test(timeout=60000) + public void testMergeTwoRegions() throws Exception { + final TableName tableName = TableName.valueOf("testMergeTwoRegions"); + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[2][]; + splitRows[0] = new byte[]{(byte)'3'}; + splitRows[1] = new byte[]{(byte)'6'}; + admin.createTable(desc, splitRows); + + List<HRegionInfo> tableRegions; + HRegionInfo [] regionsToMerge = new HRegionInfo[2]; + + tableRegions = admin.getTableRegions(tableName); + assertEquals(3, admin.getTableRegions(tableName).size()); + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure(new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge, true)); + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertEquals(2, admin.getTableRegions(tableName).size()); + } + + /** + * This tests two concurrent region merges + */ + @Test(timeout=90000) + public void testMergeRegionsConcurrently() throws Exception { + final TableName tableName = TableName.valueOf("testMergeTwoRegions"); + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[3][]; + splitRows[0] = new byte[]{(byte)'2'}; + splitRows[1] = new byte[]{(byte)'4'}; + splitRows[2] = new byte[]{(byte)'6'}; + admin.createTable(desc, splitRows); + + List<HRegionInfo> tableRegions; + HRegionInfo [] regionsToMerge1 = new HRegionInfo[2]; + HRegionInfo [] regionsToMerge2 = new HRegionInfo[2]; + + tableRegions = admin.getTableRegions(tableName); + assertEquals(4, admin.getTableRegions(tableName).size()); + regionsToMerge1[0] = tableRegions.get(0); + regionsToMerge1[1] = tableRegions.get(1); + regionsToMerge2[0] = tableRegions.get(2); + regionsToMerge2[1] = tableRegions.get(3); + + long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge1, true)); + long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge2, true)); + ProcedureTestingUtility.waitProcedure(procExec, procId1); + ProcedureTestingUtility.waitProcedure(procExec, procId2); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); + + assertEquals(2, admin.getTableRegions(tableName).size()); + } + + @Test(timeout=60000) + public void testMergeRegionsTwiceWithSameNonce() throws Exception { + final TableName tableName = TableName.valueOf("testMergeRegionsTwiceWithSameNonce"); + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[2][]; + splitRows[0] = new byte[]{(byte)'3'}; + splitRows[1] = new byte[]{(byte)'6'}; + admin.createTable(desc, splitRows); + + List<HRegionInfo> tableRegions; + HRegionInfo [] regionsToMerge = new HRegionInfo[2]; + + tableRegions = admin.getTableRegions(tableName); + assertEquals(3, admin.getTableRegions(tableName).size()); + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge, true), nonceGroup, nonce); + long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge, true), nonceGroup, nonce); + ProcedureTestingUtility.waitProcedure(procExec, procId1); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); + // The second proc should succeed too - because it is the same proc. + ProcedureTestingUtility.waitProcedure(procExec, procId2); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); + assertTrue(procId1 == procId2); + assertEquals(2, admin.getTableRegions(tableName).size()); + } + + @Test(timeout=60000) + public void testRecoveryAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution"); + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[2][]; + splitRows[0] = new byte[]{(byte)'3'}; + splitRows[1] = new byte[]{(byte)'6'}; + admin.createTable(desc, splitRows); + + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + List<HRegionInfo> tableRegions; + HRegionInfo [] regionsToMerge = new HRegionInfo[2]; + + tableRegions = admin.getTableRegions(tableName); + assertEquals(3, admin.getTableRegions(tableName).size()); + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure( + new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge, true)); + + // Restart the executor and execute the step twice + int numberOfSteps = DispatchMergingRegionsState.values().length; + MasterProcedureTestingUtility.testRecoveryAndDoubleExecution( + procExec, + procId, + numberOfSteps, + DispatchMergingRegionsState.values()); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertEquals(2, admin.getTableRegions(tableName).size()); + } + + @Test(timeout = 60000) + public void testRollbackAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution"); + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[2][]; + splitRows[0] = new byte[]{(byte)'3'}; + splitRows[1] = new byte[]{(byte)'6'}; + admin.createTable(desc, splitRows); + + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + List<HRegionInfo> tableRegions; + HRegionInfo [] regionsToMerge = new HRegionInfo[2]; + + tableRegions = admin.getTableRegions(tableName); + assertEquals(3, admin.getTableRegions(tableName).size()); + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure( + new DispatchMergingRegionsProcedure( + procExec.getEnvironment(), tableName, regionsToMerge, true)); + + int numberOfSteps = DispatchMergingRegionsState.values().length - 3; + MasterProcedureTestingUtility.testRollbackAndDoubleExecution( + procExec, + procId, + numberOfSteps, + DispatchMergingRegionsState.values()); + } + + private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() { + return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java index 4d347f2..9dd5065 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.TestMobSnapshotCloneIndependence; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; @@ -56,7 +55,6 @@ import org.apache.hadoop.hbase.util.ModifyRegionUtils; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index d8b1b2e..8c9db88 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -334,7 +334,9 @@ public class TestNamespaceAuditor { // merge the two regions final Set<String> encodedRegionNamesToMerge = Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName()); - ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(), + ADMIN.mergeRegionsAsync( + hris.get(0).getEncodedNameAsBytes(), + hris.get(1).getEncodedNameAsBytes(), false); UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() { @@ -420,7 +422,9 @@ public class TestNamespaceAuditor { regionServerObserver.failMerge(true); regionServerObserver.triggered = false; - ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(), + ADMIN.mergeRegionsAsync( + hris.get(1).getEncodedNameAsBytes(), + hris.get(2).getEncodedNameAsBytes(), false); regionServerObserver.waitUtilTriggered(); hris = ADMIN.getTableRegions(tableTwo); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 3307b73..f88c7dd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; @@ -320,6 +321,7 @@ public class TestRegionMergeTransactionOnCluster { public void testMerge() throws Exception { LOG.info("Starting testMerge"); final TableName tableName = TableName.valueOf("testMerge"); + final HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin(); try { // Create table and load data. @@ -332,7 +334,7 @@ public class TestRegionMergeTransactionOnCluster { regionStates.regionOffline(a); try { // Merge offline region. Region a is offline here - ADMIN.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); + hbaseAdmin.mergeRegionsSync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); fail("Offline regions should not be able to merge"); } catch (IOException ie) { System.out.println(ie); @@ -340,18 +342,20 @@ public class TestRegionMergeTransactionOnCluster { StringUtils.stringifyException(ie).contains("regions not online") && ie instanceof MergeRegionException); } + try { // Merge the same region: b and b. - ADMIN.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true); + hbaseAdmin.mergeRegionsSync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true); fail("A region should not be able to merge with itself, even forcifully"); } catch (IOException ie) { assertTrue("Exception should mention regions not online", StringUtils.stringifyException(ie).contains("region to itself") && ie instanceof MergeRegionException); } + try { // Merge unknown regions - ADMIN.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true); + hbaseAdmin.mergeRegionsSync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true); fail("Unknown region could not be merged"); } catch (IOException ie) { assertTrue("UnknownRegionException should be thrown", @@ -419,7 +423,7 @@ public class TestRegionMergeTransactionOnCluster { TEST_UTIL.getConnection(), tablename); HRegionInfo regionA = tableRegions.get(regionAnum).getFirst(); HRegionInfo regionB = tableRegions.get(regionBnum).getFirst(); - ADMIN.mergeRegions( + ADMIN.mergeRegionsAsync( regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); return new PairOfSameType<HRegionInfo>(regionA, regionB); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 4c2d69a..8efc2d7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -42,9 +42,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TestMobSnapshotCloneIndependence; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.procedure.TestMasterFailoverWithProcedures; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; @@ -58,7 +56,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; @@ -326,9 +323,9 @@ public class TestFlushSnapshotFromClient { int numRegions = admin.getTableRegions(TABLE_NAME).size(); int numRegionsAfterMerge = numRegions - 2; - admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(), regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(), regions.get(6).getEncodedNameAsBytes(), true); // Verify that there's one region less @@ -367,9 +364,9 @@ public class TestFlushSnapshotFromClient { int numRegions = admin.getTableRegions(TABLE_NAME).size(); int numRegionsAfterMerge = numRegions - 2; - admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(), regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), + admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(), regions.get(6).getEncodedNameAsBytes(), true); waitRegionsAfterMerge(numRegionsAfterMerge); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java index 1fa681a..e194bf7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java @@ -26,13 +26,11 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.procedure.TestMasterFailoverWithProcedures; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Rule; import org.junit.experimental.categories.Category; import org.junit.rules.TestRule; http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java index 84ef6da..ecfe521 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java @@ -631,7 +631,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck { assertNotEquals(region1, region2); // do a region merge - admin.mergeRegions(region1.getEncodedNameAsBytes(), region2.getEncodedNameAsBytes(), false); + admin.mergeRegionsAsync( + region1.getEncodedNameAsBytes(), region2.getEncodedNameAsBytes(), false); // wait until region merged long timeout = System.currentTimeMillis() + 30 * 1000;
