HBASE-16119 Procedure v2 - Reimplement Merge region (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a240778 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a240778 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a240778 Branch: refs/heads/master Commit: 0a2407784146fbbe24d9ef0101599591a5aa9c21 Parents: 00b3024 Author: Stephen Yuan Jiang <syuanjiang...@gmail.com> Authored: Thu Dec 1 22:41:15 2016 -0800 Committer: Stephen Yuan Jiang <syuanjiang...@gmail.com> Committed: Thu Dec 1 22:41:15 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/client/Admin.java | 13 + .../hbase/client/ConnectionImplementation.java | 7 + .../apache/hadoop/hbase/client/HBaseAdmin.java | 96 +- .../hbase/shaded/protobuf/ProtobufUtil.java | 39 +- .../hbase/shaded/protobuf/RequestConverter.java | 17 + .../shaded/protobuf/ResponseConverter.java | 4 +- .../procedure2/ProcedureTestingUtility.java | 3 +- .../shaded/protobuf/generated/AdminProtos.java | 891 +++-- .../generated/MasterProcedureProtos.java | 1896 +++++++++-- .../shaded/protobuf/generated/MasterProtos.java | 3144 +++++++++++++----- .../src/main/protobuf/Admin.proto | 14 +- .../src/main/protobuf/Master.proto | 18 + .../src/main/protobuf/MasterProcedure.proto | 23 +- .../hbase/rsgroup/RSGroupAdminEndpoint.java | 47 +- .../BaseMasterAndRegionObserver.java | 48 +- .../hbase/coprocessor/BaseMasterObserver.java | 48 +- .../hbase/coprocessor/MasterObserver.java | 87 +- .../hadoop/hbase/master/AssignmentManager.java | 33 + .../org/apache/hadoop/hbase/master/HMaster.java | 45 + .../hbase/master/MasterCoprocessorHost.java | 123 +- .../hadoop/hbase/master/MasterRpcServices.java | 40 + .../hadoop/hbase/master/MasterServices.java | 17 +- .../hadoop/hbase/master/RegionStates.java | 8 + .../hadoop/hbase/master/ServerManager.java | 13 +- .../procedure/MergeTableRegionsProcedure.java | 907 +++++ .../procedure/SplitTableRegionProcedure.java | 20 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +- .../hbase/regionserver/HRegionServer.java | 85 +- .../hbase/regionserver/RSRpcServices.java | 31 +- .../apache/hadoop/hbase/client/TestAdmin1.java | 9 +- .../hbase/coprocessor/TestMasterObserver.java | 59 +- .../coprocessor/TestRegionServerObserver.java | 2 + .../hbase/master/MockNoopMasterServices.java | 9 + .../hadoop/hbase/master/MockRegionServer.java | 8 +- .../TestMergeTableRegionsProcedure.java | 263 ++ .../TestSplitTableRegionProcedure.java | 5 +- .../hbase/namespace/TestNamespaceAuditor.java | 51 +- .../replication/TestSerialReplication.java | 4 +- 38 files changed, 6600 insertions(+), 1537 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 52b935f..5b53a7e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -907,6 +907,7 @@ public interface Admin extends Abortable, Closeable { void mergeRegions(final byte[] nameOfRegionA, final byte[] nameOfRegionB, final boolean forcible) throws IOException; + /** * Merge two regions. Asynchronous operation. * @@ -922,6 +923,18 @@ public interface Admin extends Abortable, Closeable { final boolean forcible) throws IOException; /** + * Merge regions. Asynchronous operation. + * + * @param nameofRegionsToMerge encoded or full name of daughter regions + * @param forcible true if do a compulsory merge, otherwise we will only merge + * adjacent regions + * @throws IOException + */ + Future<Void> mergeRegionsAsync( + final byte[][] nameofRegionsToMerge, + final boolean forcible) throws IOException; + + /** * Split a table. Asynchronous operation. * * @param tableName table to split http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index aa984b1..e75d9a5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -1274,6 +1274,13 @@ class ConnectionImplementation implements ClusterConnection, Closeable { } @Override + public MasterProtos.MergeTableRegionsResponse mergeTableRegions( + RpcController controller, MasterProtos.MergeTableRegionsRequest request) + throws ServiceException { + return stub.mergeTableRegions(controller, request); + } + + @Override public MasterProtos.AssignRegionResponse assignRegion(RpcController controller, MasterProtos.AssignRegionRequest request) throws ServiceException { return stub.assignRegion(controller, request); http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 00463f38..9bfe276 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -119,8 +119,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteTabl import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DisableTableRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DisableTableResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureRequest; @@ -147,6 +145,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListTableD import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MajorCompactionTimestampForRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MajorCompactionTimestampRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest; @@ -1513,68 +1513,80 @@ public class HBaseAdmin implements Admin { final byte[] nameOfRegionA, final byte[] nameOfRegionB, final boolean forcible) throws IOException { + byte[][] nameofRegionsToMerge = new byte[2][]; + nameofRegionsToMerge[0] = nameOfRegionA; + nameofRegionsToMerge[1] = nameOfRegionB; + return mergeRegionsAsync(nameofRegionsToMerge, forcible); + } - final byte[] encodedNameOfRegionA = isEncodedRegionName(nameOfRegionA) ? - nameOfRegionA : HRegionInfo.encodeRegionName(nameOfRegionA).getBytes(); - final byte[] encodedNameOfRegionB = isEncodedRegionName(nameOfRegionB) ? - nameOfRegionB : HRegionInfo.encodeRegionName(nameOfRegionB).getBytes(); - - TableName tableName; - Pair<HRegionInfo, ServerName> pair = getRegion(nameOfRegionA); - - if (pair != null) { - if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { - throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly"); - } - tableName = pair.getFirst().getTable(); - } else { - throw new UnknownRegionException ( - "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionA)); + /** + * Merge two regions. Asynchronous operation. + * @param nameofRegionsToMerge encoded or full name of daughter regions + * @param forcible true if do a compulsory merge, otherwise we will only merge + * adjacent regions + * @throws IOException + */ + @Override + public Future<Void> mergeRegionsAsync( + final byte[][] nameofRegionsToMerge, + final boolean forcible) throws IOException { + assert(nameofRegionsToMerge.length >= 2); + byte[][] encodedNameofRegionsToMerge = new byte[nameofRegionsToMerge.length][]; + for(int i = 0; i < nameofRegionsToMerge.length; i++) { + encodedNameofRegionsToMerge[i] = isEncodedRegionName(nameofRegionsToMerge[i]) ? + nameofRegionsToMerge[i] : HRegionInfo.encodeRegionName(nameofRegionsToMerge[i]).getBytes(); } - pair = getRegion(nameOfRegionB); - if (pair != null) { - if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { - throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly"); - } + TableName tableName = null; + Pair<HRegionInfo, ServerName> pair; - if (!tableName.equals(pair.getFirst().getTable())) { - throw new IllegalArgumentException ("Cannot merge regions from two different tables " + - tableName + " and " + pair.getFirst().getTable()); + for(int i = 0; i < nameofRegionsToMerge.length; i++) { + pair = getRegion(nameofRegionsToMerge[i]); + + if (pair != null) { + if (pair.getFirst().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { + throw new IllegalArgumentException ("Can't invoke merge on non-default regions directly"); + } + if (tableName == null) { + tableName = pair.getFirst().getTable(); + } else if (!tableName.equals(pair.getFirst().getTable())) { + throw new IllegalArgumentException ("Cannot merge regions from two different tables " + + tableName + " and " + pair.getFirst().getTable()); + } + } else { + throw new UnknownRegionException ( + "Can't invoke merge on unknown region " + + Bytes.toStringBinary(encodedNameofRegionsToMerge[i])); } - } else { - throw new UnknownRegionException ( - "Can't invoke merge on unknown region " + Bytes.toStringBinary(encodedNameOfRegionB)); } - DispatchMergingRegionsResponse response = - executeCallable(new MasterCallable<DispatchMergingRegionsResponse>(getConnection(), + MergeTableRegionsResponse response = + executeCallable(new MasterCallable<MergeTableRegionsResponse>(getConnection(), getRpcControllerFactory()) { @Override - protected DispatchMergingRegionsResponse rpcCall() throws Exception { - DispatchMergingRegionsRequest request = RequestConverter - .buildDispatchMergingRegionsRequest( - encodedNameOfRegionA, - encodedNameOfRegionB, + protected MergeTableRegionsResponse rpcCall() throws Exception { + MergeTableRegionsRequest request = RequestConverter + .buildMergeTableRegionsRequest( + encodedNameofRegionsToMerge, forcible, ng.getNonceGroup(), ng.newNonce()); - return master.dispatchMergingRegions(getRpcController(), request); + return master.mergeTableRegions(getRpcController(), request); } }); - return new DispatchMergingRegionsFuture(this, tableName, response); + return new MergeTableRegionsFuture(this, tableName, response); } - private static class DispatchMergingRegionsFuture extends TableFuture<Void> { - public DispatchMergingRegionsFuture( + private static class MergeTableRegionsFuture extends TableFuture<Void> { + public MergeTableRegionsFuture( final HBaseAdmin admin, final TableName tableName, - final DispatchMergingRegionsResponse response) { + final MergeTableRegionsResponse response) { super(admin, tableName, (response != null && response.hasProcId()) ? response.getProcId() : null); } - public DispatchMergingRegionsFuture( + public MergeTableRegionsFuture( final HBaseAdmin admin, final TableName tableName, final Long procId) { http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 5876fae..c6d647e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -105,8 +105,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; @@ -1756,26 +1756,26 @@ public final class ProtobufUtil { } /** - * A helper to close a region for split + * A helper to close a region for split or merge * using admin protocol. * * @param controller RPC controller * @param admin Admin service * @param server the RS that hosts the target region - * @param parentRegionInfo the target region info + * @param regionInfo the target region info * @return true if the region is closed * @throws IOException */ - public static boolean closeRegionForSplit( + public static boolean closeRegionForSplitOrMerge( final RpcController controller, final AdminService.BlockingInterface admin, final ServerName server, - final HRegionInfo parentRegionInfo) throws IOException { - CloseRegionForSplitRequest closeRegionForSplitRequest = - ProtobufUtil.buildCloseRegionForSplitRequest(server, parentRegionInfo); + final HRegionInfo... regionInfo) throws IOException { + CloseRegionForSplitOrMergeRequest closeRegionForRequest = + ProtobufUtil.buildCloseRegionForSplitOrMergeRequest(server, regionInfo); try { - CloseRegionForSplitResponse response = - admin.closeRegionForSplit(controller, closeRegionForSplitRequest); + CloseRegionForSplitOrMergeResponse response = + admin.closeRegionForSplitOrMerge(controller, closeRegionForRequest); return ResponseConverter.isClosed(response); } catch (ServiceException se) { throw getRemoteException(se); @@ -3130,19 +3130,22 @@ public final class ProtobufUtil { } /** - * Create a CloseRegionForSplitRequest for a given region + * Create a CloseRegionForSplitOrMergeRequest for given regions * * @param server the RS server that hosts the region - * @param parentRegionInfo the info of the region to close + * @param regionsToClose the info of the regions to close * @return a CloseRegionForSplitRequest */ - public static CloseRegionForSplitRequest buildCloseRegionForSplitRequest( + public static CloseRegionForSplitOrMergeRequest buildCloseRegionForSplitOrMergeRequest( final ServerName server, - final HRegionInfo parentRegionInfo) { - CloseRegionForSplitRequest.Builder builder = CloseRegionForSplitRequest.newBuilder(); - RegionSpecifier parentRegion = RequestConverter.buildRegionSpecifier( - RegionSpecifierType.REGION_NAME, parentRegionInfo.getRegionName()); - builder.setRegion(parentRegion); + final HRegionInfo... regionsToClose) { + CloseRegionForSplitOrMergeRequest.Builder builder = + CloseRegionForSplitOrMergeRequest.newBuilder(); + for(int i = 0; i < regionsToClose.length; i++) { + RegionSpecifier regionToClose = RequestConverter.buildRegionSpecifier( + RegionSpecifierType.REGION_NAME, regionsToClose[i].getRegionName()); + builder.addRegion(regionToClose); + } return builder.build(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 306c237..f938fd0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsCatalogJ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsNormalizerEnabledRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest; @@ -1096,6 +1097,22 @@ public final class RequestConverter { return builder.build(); } + public static MergeTableRegionsRequest buildMergeTableRegionsRequest( + final byte[][] encodedNameOfdaughaterRegions, + final boolean forcible, + final long nonceGroup, + final long nonce) throws DeserializationException { + MergeTableRegionsRequest.Builder builder = MergeTableRegionsRequest.newBuilder(); + for (int i = 0; i< encodedNameOfdaughaterRegions.length; i++) { + builder.addRegion(buildRegionSpecifier( + RegionSpecifierType.ENCODED_REGION_NAME, encodedNameOfdaughaterRegions[i])); + } + builder.setForcible(forcible); + builder.setNonceGroup(nonceGroup); + builder.setNonce(nonce); + return builder.build(); + } + public static SplitTableRegionRequest buildSplitTableRegionRequest( final HRegionInfo regionInfo, final byte[] splitPoint, http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java index 11fc931..760f630 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.SingleResponse; import org.apache.hadoop.hbase.ipc.ServerRpcController; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse; @@ -259,7 +259,7 @@ public final class ResponseConverter { * @return the region close state */ public static boolean isClosed - (final CloseRegionForSplitResponse proto) { + (final CloseRegionForSplitOrMergeResponse proto) { if (proto == null || !proto.hasClosed()) return false; return proto.getClosed(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/0a240778/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java index 8c9cea2..93f3460 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java @@ -149,7 +149,8 @@ public class ProcedureTestingUtility { assertSingleExecutorForKillTests(procExecutor); } - private static <TEnv> void assertSingleExecutorForKillTests(final ProcedureExecutor<TEnv> procExecutor) { + private static <TEnv> void assertSingleExecutorForKillTests( + final ProcedureExecutor<TEnv> procExecutor) { if (procExecutor.testing == null) return; if (procExecutor.testing.killBeforeStoreUpdate || procExecutor.testing.toggleKillBeforeStoreUpdate) {
