This is an automated email from the ASF dual-hosted git repository. wchevreuil pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push: new a208ba2a141 HBASE-28064:Implement truncate_region command (#5480) a208ba2a141 is described below commit a208ba2a1412eec9bd3500df361c261d24c78b3f Author: Vaibhav Joshi <vjo...@cloudera.com> AuthorDate: Tue Oct 31 16:13:35 2023 +0530 HBASE-28064:Implement truncate_region command (#5480) Signed-off-by: Wellington Chevreuil <wchevre...@apache.org> --- .../java/org/apache/hadoop/hbase/client/Admin.java | 14 ++ .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 6 + .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 + .../hbase/client/ConnectionImplementation.java | 6 + .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 59 ++++++ .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 66 +++++++ .../hbase/client/ShortCircuitMasterConnection.java | 6 + .../hbase/shaded/protobuf/RequestConverter.java | 11 ++ .../src/main/protobuf/Master.proto | 16 ++ .../src/main/protobuf/MasterProcedure.proto | 8 + .../hadoop/hbase/coprocessor/MasterObserver.java | 40 ++++ .../org/apache/hadoop/hbase/master/HMaster.java | 31 +++ .../hadoop/hbase/master/MasterCoprocessorHost.java | 54 +++++ .../hadoop/hbase/master/MasterRpcServices.java | 12 ++ .../apache/hadoop/hbase/master/MasterServices.java | 9 + .../hbase/master/assignment/AssignmentManager.java | 6 + .../AbstractStateMachineRegionProcedure.java | 6 + .../master/procedure/TableProcedureInterface.java | 5 +- .../hadoop/hbase/master/procedure/TableQueue.java | 1 + .../master/procedure/TruncateRegionProcedure.java | 219 +++++++++++++++++++++ .../org/apache/hadoop/hbase/client/TestAdmin1.java | 94 ++++++++- .../hbase/client/TestAsyncRegionAdminApi2.java | 83 ++++++++ .../hbase/master/MockNoopMasterServices.java | 6 + .../procedure/TestTruncateRegionProcedure.java | 202 +++++++++++++++++++ hbase-shell/src/main/ruby/hbase/admin.rb | 10 + hbase-shell/src/main/ruby/shell.rb | 1 + .../main/ruby/shell/commands/truncate_region.rb | 36 ++++ hbase-shell/src/test/ruby/hbase/admin_test.rb | 11 ++ .../hadoop/hbase/thrift2/client/ThriftAdmin.java | 10 + 29 files changed, 1030 insertions(+), 3 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 3fc844b1aac..ab3df597bfa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -3303,4 +3303,18 @@ public interface Admin extends Abortable, Closeable { * Flush master local region */ void flushMasterStore() throws IOException; + + /** + * Truncate an individual region. + * @param regionName region to truncate + * @throws IOException if a remote or network exception occurs + */ + void truncateRegion(byte[] regionName) throws IOException; + + /** + * Truncate an individual region. Asynchronous operation. + * @param regionName region to truncate + * @throws IOException if a remote or network exception occurs + */ + Future<Void> truncateRegionAsync(byte[] regionName) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 9509a888cc5..6b95e19a22a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -625,6 +625,12 @@ public interface AsyncAdmin { */ CompletableFuture<Void> splitRegion(byte[] regionName, byte[] splitPoint); + /** + * Truncate an individual region. + * @param regionName region to truncate + */ + CompletableFuture<Void> truncateRegion(byte[] regionName); + /** * Assign an individual region. * @param regionName Encoded or full name of region to assign. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 442fb911556..4c9138e7306 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -386,6 +386,11 @@ class AsyncHBaseAdmin implements AsyncAdmin { return wrap(rawAdmin.splitRegion(regionName, splitPoint)); } + @Override + public CompletableFuture<Void> truncateRegion(byte[] regionName) { + return wrap(rawAdmin.truncateRegion(regionName)); + } + @Override public CompletableFuture<Void> assign(byte[] regionName) { return wrap(rawAdmin.assign(regionName)); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 70d5760df48..268a2495d69 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -1559,6 +1559,12 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { return stub.splitRegion(controller, request); } + @Override + public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller, + MasterProtos.TruncateRegionRequest request) throws ServiceException { + return stub.truncateRegion(controller, request); + } + @Override public MasterProtos.DeleteTableResponse deleteTable(RpcController controller, MasterProtos.DeleteTableRequest request) throws ServiceException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index b4e9390c13e..c54e561cc21 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2057,6 +2057,65 @@ public class HBaseAdmin implements Admin { splitRegionAsync(regionServerPair.getFirst(), splitPoint); } + @Override + public void truncateRegion(byte[] regionName) throws IOException { + get(truncateRegionAsync(regionName), syncWaitTimeout, TimeUnit.MILLISECONDS); + } + + @Override + public Future<Void> truncateRegionAsync(byte[] regionName) throws IOException { + Pair<RegionInfo, ServerName> pair = getRegion(regionName); + RegionInfo hri; + + if (pair != null) { + hri = pair.getFirst(); + if (hri != null && hri.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { + throw new IllegalArgumentException( + "Can't truncate replicas directly.Replicas are auto-truncated " + + "when their primary is truncated."); + } + } else { + throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName)); + } + + TableName tableName = hri.getTable(); + + MasterProtos.TruncateRegionResponse response = + executeCallable(getTruncateRegionCallable(tableName, hri)); + + return new TruncateRegionFuture(this, tableName, response); + } + + private MasterCallable<MasterProtos.TruncateRegionResponse> + getTruncateRegionCallable(TableName tableName, RegionInfo hri) { + return new MasterCallable<MasterProtos.TruncateRegionResponse>(getConnection(), + getRpcControllerFactory()) { + Long nonceGroup = ng.getNonceGroup(); + Long nonce = ng.newNonce(); + + @Override + protected MasterProtos.TruncateRegionResponse rpcCall() throws Exception { + setPriority(tableName); + MasterProtos.TruncateRegionRequest request = + RequestConverter.buildTruncateRegionRequest(hri, nonceGroup, nonce); + return master.truncateRegion(getRpcController(), request); + } + }; + } + + private static class TruncateRegionFuture extends TableFuture<Void> { + public TruncateRegionFuture(final HBaseAdmin admin, final TableName tableName, + final MasterProtos.TruncateRegionResponse response) { + super(admin, tableName, + (response != null && response.hasProcId()) ? response.getProcId() : null); + } + + @Override + public String getOperationType() { + return "TRUNCATE_REGION"; + } + } + private static class ModifyTableFuture extends TableFuture<Void> { public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName, final ModifyTableResponse response) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 9231c423ce6..7a560e51e0a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -1603,6 +1603,60 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return future; } + @Override + public CompletableFuture<Void> truncateRegion(byte[] regionName) { + CompletableFuture<Void> future = new CompletableFuture<>(); + addListener(getRegionLocation(regionName), (location, err) -> { + if (err != null) { + future.completeExceptionally(err); + return; + } + RegionInfo regionInfo = location.getRegion(); + if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { + future.completeExceptionally(new IllegalArgumentException( + "Can't truncate replicas directly.Replicas are auto-truncated " + + "when their primary is truncated.")); + return; + } + ServerName serverName = location.getServerName(); + if (serverName == null) { + future + .completeExceptionally(new NoServerForRegionException(Bytes.toStringBinary(regionName))); + return; + } + addListener(truncateRegion(regionInfo), (ret, err2) -> { + if (err2 != null) { + future.completeExceptionally(err2); + } else { + future.complete(ret); + } + }); + }); + return future; + } + + private CompletableFuture<Void> truncateRegion(final RegionInfo hri) { + CompletableFuture<Void> future = new CompletableFuture<>(); + TableName tableName = hri.getTable(); + final MasterProtos.TruncateRegionRequest request; + try { + request = RequestConverter.buildTruncateRegionRequest(hri, ng.getNonceGroup(), ng.newNonce()); + } catch (DeserializationException e) { + future.completeExceptionally(e); + return future; + } + addListener(this.procedureCall(tableName, request, MasterService.Interface::truncateRegion, + MasterProtos.TruncateRegionResponse::getProcId, + new TruncateRegionProcedureBiConsumer(tableName)), (ret, err2) -> { + if (err2 != null) { + future.completeExceptionally(err2); + } else { + future.complete(ret); + } + }); + return future; + } + @Override public CompletableFuture<Void> assign(byte[] regionName) { CompletableFuture<Void> future = new CompletableFuture<>(); @@ -2850,6 +2904,18 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } } + private static class TruncateRegionProcedureBiConsumer extends TableProcedureBiConsumer { + + TruncateRegionProcedureBiConsumer(TableName tableName) { + super(tableName); + } + + @Override + String getOperationType() { + return "TRUNCATE_REGION"; + } + } + private static class SnapshotProcedureBiConsumer extends TableProcedureBiConsumer { SnapshotProcedureBiConsumer(TableName tableName) { super(tableName); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java index ea874742041..57f3cba8605 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java @@ -730,6 +730,12 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection { return stub.splitRegion(controller, request); } + @Override + public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller, + MasterProtos.TruncateRegionRequest request) throws ServiceException { + return stub.truncateRegion(controller, request); + } + @Override public SwitchRpcThrottleResponse switchRpcThrottle(RpcController controller, SwitchRpcThrottleRequest request) throws ServiceException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index d6956f4e9df..cd6fd2bb9c4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1187,6 +1187,17 @@ public final class RequestConverter { return builder.build(); } + public static MasterProtos.TruncateRegionRequest + buildTruncateRegionRequest(final RegionInfo regionInfo, final long nonceGroup, final long nonce) + throws DeserializationException { + MasterProtos.TruncateRegionRequest.Builder builder = + MasterProtos.TruncateRegionRequest.newBuilder(); + builder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo)); + builder.setNonceGroup(nonceGroup); + builder.setNonce(nonce); + return builder.build(); + } + /** * Create a protocol buffer AssignRegionRequest * @return an AssignRegionRequest diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 6ca065dfc94..ce736cd799e 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -136,6 +136,16 @@ message SplitTableRegionResponse { optional uint64 proc_id = 1; } +message TruncateRegionRequest { + required RegionInfo region_info = 1; + optional uint64 nonce_group = 2 [default = 0]; + optional uint64 nonce = 3 [default = 0]; +} + +message TruncateRegionResponse { + optional uint64 proc_id = 1; +} + message CreateTableRequest { required TableSchema table_schema = 1; repeated bytes split_keys = 2; @@ -863,6 +873,12 @@ service MasterService { rpc SplitRegion(SplitTableRegionRequest) returns(SplitTableRegionResponse); + /** + * Truncate region + */ + rpc TruncateRegion(TruncateRegionRequest) + returns(TruncateRegionResponse); + /** Deletes a table */ rpc DeleteTable(DeleteTableRequest) returns(DeleteTableResponse); diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 90d0341df64..b81507269ae 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -102,6 +102,14 @@ message TruncateTableStateData { repeated RegionInfo region_info = 5; } +enum TruncateRegionState { + TRUNCATE_REGION_PRE_OPERATION = 1; + TRUNCATE_REGION_MAKE_OFFLINE = 2; + TRUNCATE_REGION_REMOVE = 3; + TRUNCATE_REGION_MAKE_ONLINE = 4; + TRUNCATE_REGION_POST_OPERATION = 5; +} + enum DeleteTableState { DELETE_TABLE_PRE_OPERATION = 1; DELETE_TABLE_REMOVE_FROM_META = 2; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index a49f5764843..5de99adf885 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -665,6 +665,46 @@ public interface MasterObserver { final TableName tableName, final byte[] splitRow) throws IOException { } + /** + * Called before the region is truncated. + * @param c The environment to interact with the framework and master + * @param regionInfo The Region being truncated + */ + @SuppressWarnings("unused") + default void preTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c, + final RegionInfo regionInfo) { + } + + /** + * Called before the truncate region procedure is called. + * @param c The environment to interact with the framework and master + * @param regionInfo The Region being truncated + */ + @SuppressWarnings("unused") + default void preTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c, + RegionInfo regionInfo) { + } + + /** + * Called after the truncate region procedure is called. + * @param c The environment to interact with the framework and master + * @param regionInfo The Region being truncated + */ + @SuppressWarnings("unused") + default void postTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c, + RegionInfo regionInfo) { + } + + /** + * Called post the region is truncated. + * @param c The environment to interact with the framework and master + * @param regionInfo The Region To be truncated + */ + @SuppressWarnings("unused") + default void postTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c, + final RegionInfo regionInfo) { + } + /** * Called after the region is split. * @param c the environment to interact with the framework and master diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 95ce05c6ef4..c92b3559a90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -163,6 +163,7 @@ import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure; import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.master.region.MasterRegionFactory; @@ -2493,6 +2494,36 @@ public class HMaster extends HRegionServer implements MasterServices { }); } + @Override + public long truncateRegion(final RegionInfo regionInfo, final long nonceGroup, final long nonce) + throws IOException { + checkInitialized(); + + return MasterProcedureUtil + .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) { + @Override + protected void run() throws IOException { + getMaster().getMasterCoprocessorHost().preTruncateRegion(regionInfo); + + LOG.info( + getClientIdAuditPrefix() + " truncate region " + regionInfo.getRegionNameAsString()); + + // Execute the operation asynchronously + ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch(2, 0); + submitProcedure( + new TruncateRegionProcedure(procedureExecutor.getEnvironment(), regionInfo, latch)); + latch.await(); + + getMaster().getMasterCoprocessorHost().postTruncateRegion(regionInfo); + } + + @Override + protected String getDescription() { + return "TruncateRegionProcedure"; + } + }); + } + @Override public long addColumn(final TableName tableName, final ColumnFamilyDescriptor column, final long nonceGroup, final long nonce) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index aa20c2afea2..95c183b4a39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -862,6 +862,60 @@ public class MasterCoprocessorHost }); } + /** + * Invoked just before calling the truncate region procedure + * @param regionInfo region being truncated + */ + public void preTruncateRegion(RegionInfo regionInfo) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) { + observer.preTruncateRegion(this, regionInfo); + } + }); + } + + /** + * Invoked after calling the truncate region procedure + * @param regionInfo region being truncated + */ + public void postTruncateRegion(RegionInfo regionInfo) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) { + observer.postTruncateRegion(this, regionInfo); + } + }); + } + + /** + * Invoked just before calling the truncate region procedure + * @param region Region to be truncated + * @param user The user + */ + public void preTruncateRegionAction(final RegionInfo region, User user) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preTruncateRegionAction(this, region); + } + }); + } + + /** + * Invoked after calling the truncate region procedure + * @param region Region which was truncated + * @param user The user + */ + public void postTruncateRegionAction(final RegionInfo region, User user) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postTruncateRegionAction(this, region); + } + }); + } + /** * This will be called before update META step as part of split table region procedure. * @param user the user diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 468a477288c..d4cfaaaf47a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -884,6 +884,18 @@ public class MasterRpcServices extends RSRpcServices } } + @Override + public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller, + final MasterProtos.TruncateRegionRequest request) throws ServiceException { + try { + long procId = master.truncateRegion(ProtobufUtil.toRegionInfo(request.getRegionInfo()), + request.getNonceGroup(), request.getNonce()); + return MasterProtos.TruncateRegionResponse.newBuilder().setProcId(procId).build(); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } + @Override public ClientProtos.CoprocessorServiceResponse execMasterService(final RpcController controller, final ClientProtos.CoprocessorServiceRequest request) throws ServiceException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index d465a4a6727..35044fb761a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -448,4 +448,13 @@ public interface MasterServices extends Server { */ long flushTable(final TableName tableName, final List<byte[]> columnFamilies, final long nonceGroup, final long nonce) throws IOException; + + /** + * Truncate region + * @param regionInfo region to be truncated + * @param nonceGroup the nonce group + * @param nonce the nonce + * @return procedure Id + */ + long truncateRegion(RegionInfo regionInfo, long nonceGroup, long nonce) throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 55edb5af5d5..b5892b42917 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure; import org.apache.hadoop.hbase.master.region.MasterRegion; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; @@ -1075,6 +1076,11 @@ public class AssignmentManager { return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey); } + public TruncateRegionProcedure createTruncateRegionProcedure(final RegionInfo regionToTruncate) + throws IOException { + return new TruncateRegionProcedure(getProcedureEnvironment(), regionToTruncate); + } + public MergeTableRegionsProcedure createMergeProcedure(RegionInfo... ris) throws IOException { return new MergeTableRegionsProcedure(getProcedureEnvironment(), ris, false); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java index fe98a78b4d7..cf886e9824b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java @@ -41,6 +41,12 @@ public abstract class AbstractStateMachineRegionProcedure<TState> this.hri = hri; } + protected AbstractStateMachineRegionProcedure(MasterProcedureEnv env, RegionInfo hri, + ProcedurePrepareLatch latch) { + super(env, latch); + this.hri = hri; + } + protected AbstractStateMachineRegionProcedure() { // Required by the Procedure framework to create the procedure on replay super(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java index b5aaada9c45..1018535a173 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java @@ -42,8 +42,9 @@ public interface TableProcedureInterface { REGION_ASSIGN, REGION_UNASSIGN, REGION_GC, - MERGED_REGIONS_GC/* region operations */ - }; + MERGED_REGIONS_GC/* region operations */, + REGION_TRUNCATE + } /** Returns the name of the table the procedure is operating on */ TableName getTableName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java index d3eae050b56..910a684b51f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java @@ -69,6 +69,7 @@ class TableQueue extends Queue<TableName> { case REGION_GC: case MERGED_REGIONS_GC: case REGION_SNAPSHOT: + case REGION_TRUNCATE: return false; default: break; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java new file mode 100644 index 00000000000..5e907c1681a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateRegionState; + +@InterfaceAudience.Private +public class TruncateRegionProcedure + extends AbstractStateMachineRegionProcedure<TruncateRegionState> { + private static final Logger LOG = LoggerFactory.getLogger(TruncateRegionProcedure.class); + + @SuppressWarnings("unused") + public TruncateRegionProcedure() { + // Required by the Procedure framework to create the procedure on replay + super(); + } + + public TruncateRegionProcedure(final MasterProcedureEnv env, final RegionInfo hri) + throws HBaseIOException { + super(env, hri); + checkOnline(env, getRegion()); + } + + public TruncateRegionProcedure(final MasterProcedureEnv env, final RegionInfo region, + ProcedurePrepareLatch latch) throws HBaseIOException { + super(env, region, latch); + preflightChecks(env, true); + } + + @Override + protected Flow executeFromState(final MasterProcedureEnv env, TruncateRegionState state) + throws InterruptedException { + if (LOG.isTraceEnabled()) { + LOG.trace(this + " execute state=" + state); + } + try { + switch (state) { + case TRUNCATE_REGION_PRE_OPERATION: + if (!prepareTruncate()) { + assert isFailed() : "the truncate should have an exception here"; + return Flow.NO_MORE_STATE; + } + checkOnline(env, getRegion()); + assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed() + : "Can't truncate replicas directly. " + + "Replicas are auto-truncated when their primary is truncated."; + preTruncate(env); + setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE); + break; + case TRUNCATE_REGION_MAKE_OFFLINE: + addChildProcedure(createUnAssignProcedures(env)); + setNextState(TruncateRegionState.TRUNCATE_REGION_REMOVE); + break; + case TRUNCATE_REGION_REMOVE: + deleteRegionFromFileSystem(env); + setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_ONLINE); + break; + case TRUNCATE_REGION_MAKE_ONLINE: + addChildProcedure(createAssignProcedures(env)); + setNextState(TruncateRegionState.TRUNCATE_REGION_POST_OPERATION); + break; + case TRUNCATE_REGION_POST_OPERATION: + postTruncate(env); + LOG.debug("truncate '" + getTableName() + "' completed"); + return Flow.NO_MORE_STATE; + default: + throw new UnsupportedOperationException("unhandled state=" + state); + } + } catch (IOException e) { + if (isRollbackSupported(state)) { + setFailure("master-truncate-region", e); + } else { + LOG.warn("Retriable error trying to truncate region=" + getRegion().getRegionNameAsString() + + " state=" + state, e); + } + } + return Flow.HAS_MORE_STATE; + } + + private void deleteRegionFromFileSystem(final MasterProcedureEnv env) throws IOException { + RegionStateNode regionNode = + env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); + try { + regionNode.lock(); + final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName()); + HRegionFileSystem.deleteRegionFromFileSystem(env.getMasterConfiguration(), + mfs.getFileSystem(), tableDir, getRegion()); + } finally { + regionNode.unlock(); + } + } + + @Override + protected void rollbackState(final MasterProcedureEnv env, final TruncateRegionState state) + throws IOException { + if (state == TruncateRegionState.TRUNCATE_REGION_PRE_OPERATION) { + // Nothing to rollback, pre-truncate is just table-state checks. + return; + } + if (state == TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE) { + RegionStateNode regionNode = + env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); + if (regionNode == null) { + // Region was unassigned by state TRUNCATE_REGION_MAKE_OFFLINE. + // So Assign it back + addChildProcedure(createAssignProcedures(env)); + } + return; + } + // The truncate doesn't have a rollback. The execution will succeed, at some point. + throw new UnsupportedOperationException("unhandled state=" + state); + } + + @Override + protected void completionCleanup(final MasterProcedureEnv env) { + releaseSyncLatch(); + } + + @Override + protected boolean isRollbackSupported(final TruncateRegionState state) { + switch (state) { + case TRUNCATE_REGION_PRE_OPERATION: + return true; + case TRUNCATE_REGION_MAKE_OFFLINE: + return true; + default: + return false; + } + } + + @Override + protected TruncateRegionState getState(final int stateId) { + return TruncateRegionState.forNumber(stateId); + } + + @Override + protected int getStateId(final TruncateRegionState state) { + return state.getNumber(); + } + + @Override + protected TruncateRegionState getInitialState() { + return TruncateRegionState.TRUNCATE_REGION_PRE_OPERATION; + } + + @Override + public void toStringClassDetails(StringBuilder sb) { + sb.append(getClass().getSimpleName()); + sb.append(" (region="); + sb.append(getRegion().getRegionNameAsString()); + sb.append(")"); + } + + private boolean prepareTruncate() throws IOException { + if (getTableName().equals(TableName.META_TABLE_NAME)) { + throw new IOException("Can't truncate region in catalog tables"); + } + return true; + } + + private void preTruncate(final MasterProcedureEnv env) throws IOException { + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); + if (cpHost != null) { + cpHost.preTruncateRegionAction(getRegion(), getUser()); + } + } + + private void postTruncate(final MasterProcedureEnv env) throws IOException { + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); + if (cpHost != null) { + cpHost.postTruncateRegionAction(getRegion(), getUser()); + } + } + + @Override + public TableOperationType getTableOperationType() { + return TableOperationType.REGION_TRUNCATE; + } + + private TransitRegionStateProcedure createUnAssignProcedures(MasterProcedureEnv env) + throws IOException { + return env.getAssignmentManager().createOneUnassignProcedure(getRegion(), true); + } + + private TransitRegionStateProcedure createAssignProcedures(MasterProcedureEnv env) { + return env.getAssignmentManager().createOneAssignProcedure(getRegion(), true); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index ea28a6aa0df..4b022d72e7d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; import org.apache.hadoop.hbase.master.janitor.CatalogJanitor; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -83,7 +85,7 @@ public class TestAdmin1 extends TestAdminBase { Region metaRegion = null; for (int i = 0; i < NB_SERVERS; i++) { HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); - List<HRegion> onlineRegions = rs.getRegions(TableName.META_TABLE_NAME); + List<HRegion> onlineRegions = rs.getRegions(META_TABLE_NAME); if (!onlineRegions.isEmpty()) { metaRegion = onlineRegions.get(0); break; @@ -682,4 +684,94 @@ public class TestAdmin1 extends TestAdminBase { MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName, true); assertEquals(1, allRegions.size()); } + + @Test + public void testTruncateRegions() throws Exception { + // Arrange - Create table, insert data, identify region to truncate. + final TableName tableName = TableName.valueOf(name.getMethodName()); + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") }; + String family1 = "f1"; + String family2 = "f2"; + final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) }; + final String[] sFamilies = new String[] { family1, family2 }; + int replicaCount = 2; + try { + TEST_UTIL.createTable(tableName, bFamilies, splitKeys, replicaCount); + TEST_UTIL.waitTableAvailable(tableName); + + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies); + + List<RegionInfo> tableRegions = ADMIN.getRegions(tableName); + + RegionInfo regionToBeTruncated = tableRegions.get(0); + int countBeforeTruncate = TEST_UTIL.countRows(tableName); + + // Act - Truncate the first region + ADMIN.truncateRegion(regionToBeTruncated.getRegionName()); + + int countAfterTruncate = TEST_UTIL.countRows(tableName); + + // Assert - Assert that before truncate count was 8 and after truncate its 6 + assertEquals(8, countBeforeTruncate); + assertEquals(6, countAfterTruncate); + } finally { + ADMIN.disableTable(tableName); + ADMIN.deleteTable(tableName); + } + } + + @Test + public void testTruncateReplicaRegionNotAllowed() throws Exception { + // Arrange - Create table, insert data, identify region to truncate. + final TableName tableName = TableName.valueOf(name.getMethodName()); + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") }; + String family1 = "f1"; + String family2 = "f2"; + final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) }; + int replicaCount = 2; + try { + TEST_UTIL.createTable(tableName, bFamilies, splitKeys, replicaCount); + TEST_UTIL.waitTableAvailable(tableName); + + List<RegionInfo> tableRegions = ADMIN.getRegions(tableName); + + RegionInfo firstRegion = tableRegions.get(0); + RegionInfo regionToBeTruncated = RegionReplicaUtil.getRegionInfoForReplica(firstRegion, 1); + + // Act - Truncate the first region replica + try { + ADMIN.truncateRegion(regionToBeTruncated.getRegionName()); + } catch (Exception e) { + // Assert + assertEquals("Expected message is different", + "Can't truncate replicas directly.Replicas are auto-truncated " + + "when their primary is truncated.", + e.getMessage()); + } + } finally { + ADMIN.disableTable(tableName); + ADMIN.deleteTable(tableName); + } + } + + @Test + public void testTruncateRegionMetaTableRegionsNotAllowed() throws Exception { + // Arrange - Get the region of META table + List<RegionInfo> regions = ADMIN.getRegions(META_TABLE_NAME); + RegionInfo regionToBeTruncated = regions.get(0); + + // Act + try { + ADMIN.truncateRegion(regionToBeTruncated.getRegionName()); + } catch (Exception e) { + String expectedErrorMessage = + "Invalid region: " + Bytes.toStringBinary(regionToBeTruncated.getRegionName()); + assertEquals(expectedErrorMessage, e.getMessage()); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java index 44106fb4ccf..b31369cbdd3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java @@ -22,10 +22,12 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -37,6 +39,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; import org.apache.hadoop.hbase.master.janitor.CatalogJanitor; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -283,4 +286,84 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { } assertEquals(2, count); } + + @Test + public void testTruncateRegion() throws Exception { + // Arrange - Create table, insert data, identify region to truncate. + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") }; + String family1 = "f1"; + String family2 = "f2"; + + final String[] sFamilies = new String[] { family1, family2 }; + final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) }; + createTableWithDefaultConf(tableName, splitKeys, bFamilies); + + AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); + List<HRegionLocation> regionLocations = + AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); + RegionInfo regionToBeTruncated = regionLocations.get(0).getRegion(); + + assertEquals(4, regionLocations.size()); + + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies); + AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies); + int rowCountBeforeTruncate = TEST_UTIL.countRows(tableName); + + // Act - Truncate the first region + admin.truncateRegion(regionToBeTruncated.getRegionName()).get(); + + // Assert + int rowCountAfterTruncate = TEST_UTIL.countRows(tableName); + assertNotEquals(rowCountBeforeTruncate, rowCountAfterTruncate); + int expectedRowCount = rowCountBeforeTruncate - 2;// Since region with 2 rows was truncated. + assertEquals(expectedRowCount, rowCountAfterTruncate); + } + + @Test + public void testTruncateReplicaRegionNotAllowed() throws Exception { + // Arrange - Create table, insert data, identify region to truncate. + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") }; + String family1 = "f1"; + String family2 = "f2"; + + final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) }; + createTableWithDefaultConf(tableName, 2, splitKeys, bFamilies); + + AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); + List<HRegionLocation> regionLocations = + AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get(); + RegionInfo primaryRegion = regionLocations.get(0).getRegion(); + + RegionInfo firstReplica = RegionReplicaUtil.getRegionInfoForReplica(primaryRegion, 1); + + // Act - Truncate the first region + try { + admin.truncateRegion(firstReplica.getRegionName()).get(); + } catch (Exception e) { + // Assert + assertEquals("Expected message is different", + "Can't truncate replicas directly.Replicas are auto-truncated " + + "when their primary is truncated.", + e.getCause().getMessage()); + } + } + + @Test + public void testTruncateRegionsMetaTableRegionsNotAllowed() throws Exception { + AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(META_TABLE_NAME); + List<HRegionLocation> regionLocations = locator.getAllRegionLocations().get(); + HRegionLocation regionToBeTruncated = regionLocations.get(0); + try { + admin.truncateRegion(regionToBeTruncated.getRegion().getRegionName()).get(); + fail(); + } catch (ExecutionException e) { + // expected + assertThat(e.getCause(), instanceOf(IOException.class)); + assertEquals("Can't truncate region in catalog tables", e.getCause().getMessage()); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index d78caff9a39..a0951480124 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -74,6 +74,12 @@ public class MockNoopMasterServices implements MasterServices { // no-op } + @Override + public long truncateRegion(RegionInfo regionInfo, long nonceGroup, long nonce) + throws IOException { + return 0; + } + @Override public long createTable(final TableDescriptor desc, final byte[][] splitKeys, final long nonceGroup, final long nonce) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java new file mode 100644 index 00000000000..8ce60ee5550 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.assignment.TestSplitTableRegionProcedure; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +@Category({ MasterTests.class, LargeTests.class }) +public class TestTruncateRegionProcedure extends TestTableDDLProcedureBase { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTruncateRegionProcedure.class); + private static final Logger LOG = LoggerFactory.getLogger(TestTruncateRegionProcedure.class); + + @Rule + public TestName name = new TestName(); + + private static void setupConf(Configuration conf) { + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); + conf.set("hbase.coprocessor.region.classes", + TestSplitTableRegionProcedure.RegionServerHostingReplicaSlowOpenCopro.class.getName()); + conf.setInt("hbase.client.sync.wait.timeout.msec", 60000); + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(3); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + + // Turn off balancer, so it doesn't cut in and mess up our placements. + UTIL.getAdmin().balancerSwitch(false, true); + // Turn off the meta scanner, so it doesn't remove, parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + } + + @After + public void tearDown() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) { + UTIL.deleteTable(htd.getTableName()); + } + } + + @Test + public void testTruncateRegionProcedure() throws Exception { + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + // Arrange - Load table and prepare arguments values. + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String[] families = new String[] { "f1", "f2" }; + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") }; + + MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, families); + + insertData(UTIL, tableName, 2, 20, families); + insertData(UTIL, tableName, 2, 31, families); + insertData(UTIL, tableName, 2, 61, families); + insertData(UTIL, tableName, 2, 91, families); + + assertEquals(8, UTIL.countRows(tableName)); + + int rowsBeforeDropRegion = 8; + + MasterProcedureEnv environment = procExec.getEnvironment(); + RegionInfo regionToBeTruncated = environment.getAssignmentManager().getAssignedRegions() + .stream().filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString())) + .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get(); + + // Act - Execute Truncate region procedure + long procId = + procExec.submitProcedure(new TruncateRegionProcedure(environment, regionToBeTruncated)); + ProcedureTestingUtility.waitProcedure(procExec, procId); + assertEquals(8 - 2, UTIL.countRows(tableName)); + + int rowsAfterDropRegion = UTIL.countRows(tableName); + assertTrue("Row counts after truncate region should be less than row count before it", + rowsAfterDropRegion < rowsBeforeDropRegion); + assertEquals(rowsBeforeDropRegion, rowsAfterDropRegion + 2); + + insertData(UTIL, tableName, 2, 20, families); + assertEquals(8, UTIL.countRows(tableName)); + } + + @Test + public void testTruncateRegionProcedureErrorWhenSpecifiedReplicaRegionID() throws Exception { + final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); + // Arrange - Load table and prepare arguments values. + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String[] families = new String[] { "f1", "f2" }; + createTable(tableName, families, 2); + insertData(UTIL, tableName, 2, 20, families); + insertData(UTIL, tableName, 2, 30, families); + insertData(UTIL, tableName, 2, 60, families); + + assertEquals(6, UTIL.countRows(tableName)); + + MasterProcedureEnv environment = procExec.getEnvironment(); + RegionInfo regionToBeTruncated = environment.getAssignmentManager().getAssignedRegions() + .stream().filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString())) + .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get(); + + RegionInfo replicatedRegionId = + RegionReplicaUtil.getRegionInfoForReplica(regionToBeTruncated, 1); + + // Act - Execute Truncate region procedure + long procId = + procExec.submitProcedure(new TruncateRegionProcedure(environment, replicatedRegionId)); + + ProcedureTestingUtility.waitProcedure(procExec, procId); + Procedure<MasterProcedureEnv> result = procExec.getResult(procId); + // Asserts + + assertEquals(ProcedureProtos.ProcedureState.ROLLEDBACK, result.getState()); + assertTrue(result.getException().getMessage() + .endsWith("Can't truncate replicas directly. Replicas are auto-truncated " + + "when their primary is truncated.")); + } + + private TableDescriptor tableDescriptor(final TableName tableName, String[] families, + final int replicaCount) { + return TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(replicaCount) + .setColumnFamilies(columnFamilyDescriptor(families)).build(); + } + + private List<ColumnFamilyDescriptor> columnFamilyDescriptor(String[] families) { + return Arrays.stream(families).map(ColumnFamilyDescriptorBuilder::of) + .collect(Collectors.toList()); + } + + @SuppressWarnings("SameParameterValue") + private void createTable(final TableName tableName, String[] families, final int replicaCount) + throws IOException { + UTIL.getAdmin().createTable(tableDescriptor(tableName, families, replicaCount)); + } +} diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 91e1e2b8472..2110e72d155 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -163,6 +163,16 @@ module Hbase end end + #---------------------------------------------------------------------------------------------- + # Requests a region truncate + def truncate_region(region_name) + begin + org.apache.hadoop.hbase.util.FutureUtils.get(@admin.truncateRegionAsync(region_name.to_java_bytes)) + rescue java.lang.IllegalArgumentException, org.apache.hadoop.hbase.UnknownRegionException + @admin.truncate_region(region_name.to_java_bytes) + end + end + #---------------------------------------------------------------------------------------------- # Enable/disable one split or merge switch # Returns previous switch setting. diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index e0a29b32082..3d95aa1623b 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -487,6 +487,7 @@ Shell.load_command_group( list_decommissioned_regionservers decommission_regionservers recommission_regionserver + truncate_region ], # TODO: remove older hlog_roll command aliases: { diff --git a/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb b/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb new file mode 100644 index 00000000000..b443d3210bc --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TruncateRegion < Command + def help + <<-EOF +Truncate individual region. +Examples: + truncate_region 'REGIONNAME' + truncate_region 'ENCODED_REGIONNAME' +EOF + end + def command(region_name) + admin.truncate_region(region_name) + end + end + end +end diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index e84ffd6385a..dcd3d850343 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -158,6 +158,17 @@ module Hbase #------------------------------------------------------------------------------- + define_test "truncate region should work" do + @t_name = 'hbase_shell_truncate_region' + drop_test_table(@t_name) + admin.create(@t_name, 'a', NUMREGIONS => 10, SPLITALGO => 'HexStringSplit') + r1 = command(:locate_region, @t_name, '1') + region1 = r1.getRegion.getRegionNameAsString + command(:truncate_region, region1) + end + + #------------------------------------------------------------------------------- + define_test "drop should fail on non-existent tables" do assert_raise(ArgumentError) do command(:drop, 'NOT.EXISTS') diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 05fc3c932dc..183efd6288a 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -888,6 +888,16 @@ public class ThriftAdmin implements Admin { throw new NotImplementedException("split not supported in ThriftAdmin"); } + @Override + public void truncateRegion(byte[] regionName) throws IOException { + throw new NotImplementedException("Truncate Region not supported in ThriftAdmin"); + } + + @Override + public Future<Void> truncateRegionAsync(byte[] regionName) { + throw new NotImplementedException("Truncate Region Async not supported in ThriftAdmin"); + } + @Override public void splitRegion(byte[] regionName, byte[] splitPoint) { throw new NotImplementedException("splitRegion not supported in ThriftAdmin");