HBASE-14552 Procedure V2: Reimplement DispatchMergingRegionHandler (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f04eeecf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f04eeecf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f04eeecf Branch: refs/heads/master Commit: f04eeecffc4ec20a2cfc86ef55f6b724ecbc7e5e Parents: 28d8609 Author: Stephen Yuan Jiang <syuanjiang...@gmail.com> Authored: Fri Jul 15 07:06:53 2016 -0700 Committer: Stephen Yuan Jiang <syuanjiang...@gmail.com> Committed: Fri Jul 15 07:06:53 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/client/Admin.java | 17 + .../apache/hadoop/hbase/client/HBaseAdmin.java | 104 +- .../hadoop/hbase/protobuf/ProtobufUtil.java | 8 +- ...MergeRandomAdjacentRegionsOfTableAction.java | 2 +- .../generated/MasterProcedureProtos.java | 1823 +++++++++++++++--- .../hbase/protobuf/generated/MasterProtos.java | 893 ++++++--- hbase-protocol/src/main/protobuf/Master.proto | 3 + .../src/main/protobuf/MasterProcedure.proto | 15 + .../org/apache/hadoop/hbase/master/HMaster.java | 57 +- .../hadoop/hbase/master/MasterRpcServices.java | 41 +- .../hadoop/hbase/master/MasterServices.java | 17 +- .../hadoop/hbase/master/RegionStates.java | 2 +- .../hadoop/hbase/master/ServerManager.java | 5 +- .../handler/DispatchMergingRegionHandler.java | 191 -- .../normalizer/MergeNormalizationPlan.java | 2 +- .../DispatchMergingRegionsProcedure.java | 584 ++++++ .../apache/hadoop/hbase/client/TestAdmin1.java | 7 +- .../hbase/client/TestHBaseAdminNoCluster.java | 14 - .../hbase/client/TestSplitOrMergeStatus.java | 4 +- .../hbase/coprocessor/TestMasterObserver.java | 2 +- .../coprocessor/TestRegionServerObserver.java | 6 +- .../hbase/master/MockNoopMasterServices.java | 14 +- .../hbase/master/TestAssignmentListener.java | 2 +- .../TestDispatchMergingRegionsProcedure.java | 296 +++ .../TestMasterFailoverWithProcedures.java | 2 - .../hbase/namespace/TestNamespaceAuditor.java | 8 +- .../TestRegionMergeTransactionOnCluster.java | 12 +- .../snapshot/TestFlushSnapshotFromClient.java | 11 +- .../TestMobFlushSnapshotFromClient.java | 2 - .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 3 +- 30 files changed, 3320 insertions(+), 827 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 51a26bc..34e0a89 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -869,11 +869,28 @@ public interface Admin extends Abortable, Closeable { * @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent * regions * @throws IOException + * @deprecated Since 2.0. Will be removed in 3.0. Use + * {@link #mergeRegionsAsync(byte[], byte[], boolean)} instead. */ + @Deprecated void mergeRegions(final byte[] nameOfRegionA, final byte[] nameOfRegionB, final boolean forcible) throws IOException; /** + * Merge two regions. Asynchronous operation. + * + * @param nameOfRegionA encoded or full name of region a + * @param nameOfRegionB encoded or full name of region b + * @param forcible true if do a compulsory merge, otherwise we will only merge + * two adjacent regions + * @throws IOException + */ + public Future<Void> mergeRegionsAsync( + final byte[] nameOfRegionA, + final byte[] nameOfRegionB, + final boolean forcible) throws IOException; + + /** * Split a table. Asynchronous operation. * * @param tableName table to split http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index da0de51..d18b8b3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRespon import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest; @@ -1527,31 +1528,95 @@ public class HBaseAdmin implements Admin { } /** + * Merge two regions. Synchronous operation. + * Note: It is not feasible to predict the length of merge. + * Therefore, this is for internal testing only. + * @param nameOfRegionA encoded or full name of region a + * @param nameOfRegionB encoded or full name of region b + * @param forcible true if do a compulsory merge, otherwise we will only merge + * two adjacent regions + * @throws IOException + */ + @VisibleForTesting + public void mergeRegionsSync( + final byte[] nameOfRegionA, + final byte[] nameOfRegionB, + final boolean forcible) throws IOException { + get( + mergeRegionsAsync(nameOfRegionA, nameOfRegionB, forcible), + syncWaitTimeout, + TimeUnit.MILLISECONDS); + } + + /** * Merge two regions. Asynchronous operation. * @param nameOfRegionA encoded or full name of region a * @param nameOfRegionB encoded or full name of region b * @param forcible true if do a compulsory merge, otherwise we will only merge * two adjacent regions * @throws IOException + * @deprecated Since 2.0. Will be removed in 3.0. Use + * {@link #mergeRegionsAsync(byte[], byte[], boolean)} instead. */ + @Deprecated @Override public void mergeRegions(final byte[] nameOfRegionA, final byte[] nameOfRegionB, final boolean forcible) throws IOException { + mergeRegionsAsync(nameOfRegionA, nameOfRegionB, forcible); + } + + /** + * Merge two regions. Asynchronous operation. + * @param nameOfRegionA encoded or full name of region a + * @param nameOfRegionB encoded or full name of region b + * @param forcible true if do a compulsory merge, otherwise we will only merge + * two adjacent regions + * @throws IOException + */ + @Override + public Future<Void> mergeRegionsAsync( + final byte[] nameOfRegionA, + final byte[] nameOfRegionB, + final boolean forcible) throws IOException { + final byte[] encodedNameOfRegionA = isEncodedRegionName(nameOfRegionA) ? nameOfRegionA : HRegionInfo.encodeRegionName(nameOfRegionA).getBytes(); final byte[] encodedNameOfRegionB = isEncodedRegionName(nameOfRegionB) ? nameOfRegionB : HRegionInfo.encodeRegionName(nameOfRegionB).getBytes(); + TableName tableName; Pair<HRegionInfo, ServerName> pair = getRegion(nameOfRegionA); - if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) - throw new IllegalArgumentException("Can't invoke merge on non-default regions directly"); + + if (pair != null) { + if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { + throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly"); + } + tableName = pair.getFirst().getTable(); + } else { + throw new UnknownRegionException ( + "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionA)); + } + pair = getRegion(nameOfRegionB); - if (pair != null && pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) - throw new IllegalArgumentException("Can't invoke merge on non-default regions directly"); - executeCallable(new MasterCallable<Void>(getConnection()) { + if (pair != null) { + if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { + throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly"); + } + + if (!tableName.equals(pair.getFirst().getTable())) { + throw new IllegalArgumentException ("Cannot merge regions from two different tables " + + tableName + " and " + pair.getFirst().getTable()); + } + } else { + throw new UnknownRegionException ( + "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionB)); + } + + DispatchMergingRegionsResponse response = + executeCallable(new MasterCallable<DispatchMergingRegionsResponse>(getConnection()) { @Override - public Void call(int callTimeout) throws ServiceException { + public DispatchMergingRegionsResponse call(int callTimeout) throws ServiceException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); @@ -1559,13 +1624,36 @@ public class HBaseAdmin implements Admin { DispatchMergingRegionsRequest request = RequestConverter .buildDispatchMergingRegionsRequest(encodedNameOfRegionA, encodedNameOfRegionB, forcible); - master.dispatchMergingRegions(controller, request); + return master.dispatchMergingRegions(controller, request); } catch (DeserializationException de) { LOG.error("Could not parse destination server name: " + de); + throw new ServiceException(new DoNotRetryIOException(de)); } - return null; } }); + return new DispatchMergingRegionsFuture(this, tableName, response); + } + + private static class DispatchMergingRegionsFuture extends TableFuture<Void> { + public DispatchMergingRegionsFuture( + final HBaseAdmin admin, + final TableName tableName, + final DispatchMergingRegionsResponse response) { + super(admin, tableName, + (response != null && response.hasProcId()) ? response.getProcId() : null); + } + + public DispatchMergingRegionsFuture( + final HBaseAdmin admin, + final TableName tableName, + final Long procId) { + super(admin, tableName, procId); + } + + @Override + public String getOperationType() { + return "MERGE_REGIONS"; + } } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index fecc3c2..c477063 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -42,8 +42,10 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; + import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier .RegionSpecifierType.REGION_NAME; + import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; @@ -159,7 +161,6 @@ import org.apache.hadoop.hbase.quotas.ThrottleType; import org.apache.hadoop.hbase.replication.ReplicationLoadSink; import org.apache.hadoop.hbase.replication.ReplicationLoadSource; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; -import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.TablePermission; import org.apache.hadoop.hbase.security.access.UserPermission; @@ -175,6 +176,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import com.google.common.collect.ArrayListMultimap; @@ -1898,12 +1900,12 @@ public final class ProtobufUtil { public static void mergeRegions(final RpcController controller, final AdminService.BlockingInterface admin, final HRegionInfo region_a, final HRegionInfo region_b, - final boolean forcible, final User user) throws IOException { + final boolean forcible, final UserGroupInformation user) throws IOException { final MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest( region_a.getRegionName(), region_b.getRegionName(),forcible); if (user != null) { try { - user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { + user.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { admin.mergeRegions(controller, request); http://git-wip-us.apache.org/repos/asf/hbase/blob/f04eeecf/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java index 8645dc4..03d310e 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java @@ -65,7 +65,7 @@ public class MergeRandomAdjacentRegionsOfTableAction extends Action { } try { - admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); + admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); } catch (Exception ex) { LOG.warn("Merge failed, might be caused by other chaos: " + ex.getMessage()); }