HBASE-17280 Add mechanism to control hbase cleaner behavior Signed-off-by: tedyu <yuzhih...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67fe516e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67fe516e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67fe516e Branch: refs/heads/branch-1 Commit: 67fe516ec9adabc56e557cb67fb254353da4a50e Parents: e9abe07 Author: Ajay Jadhav <jadha...@amazon.com> Authored: Thu Feb 9 09:06:00 2017 -0800 Committer: tedyu <yuzhih...@gmail.com> Committed: Thu Feb 9 12:18:42 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/client/Admin.java | 24 + .../hadoop/hbase/client/ConnectionManager.java | 25 + .../apache/hadoop/hbase/client/HBaseAdmin.java | 33 + .../hadoop/hbase/protobuf/RequestConverter.java | 39 + .../hbase/protobuf/ResponseConverter.java | 9 + .../hbase/protobuf/generated/MasterProtos.java | 3669 +++++++++++++++--- hbase-protocol/src/main/protobuf/Master.proto | 38 + .../org/apache/hadoop/hbase/master/HMaster.java | 18 + .../hadoop/hbase/master/MasterRpcServices.java | 41 + .../hbase/master/cleaner/CleanerChore.java | 23 + .../hbase/master/cleaner/TestCleanerChore.java | 66 + hbase-shell/src/main/ruby/hbase/admin.rb | 20 + hbase-shell/src/main/ruby/shell.rb | 3 + .../shell/commands/cleaner_chore_enabled.rb | 40 + .../ruby/shell/commands/cleaner_chore_run.rb | 37 + .../ruby/shell/commands/cleaner_chore_switch.rb | 41 + 16 files changed, 3670 insertions(+), 456 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 270518d..5810b2b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -767,6 +767,30 @@ public interface Admin extends Abortable, Closeable { boolean isCatalogJanitorEnabled() throws IOException; /** + * Enable/Disable the cleaner chore + * + * @param on if true enables the cleaner chore + * @return the previous state + * @throws IOException + */ + public boolean setCleanerChoreRunning(final boolean on) throws IOException; + + /** + * Ask for cleaner chore to run + * + * @return True if cleaner chore ran, false otherwise + * @throws IOException + */ + public boolean runCleanerChore() throws IOException; + + /** + * Query on the cleaner chore state (Enabled/Disabled?) + * + * @throws IOException + */ + public boolean isCleanerChoreEnabled() throws IOException; + + /** * Merge two regions. Asynchronous operation. * * @param nameOfRegionA encoded or full name of region a http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index ab6cb8d..c10f893 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -128,6 +128,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsBalancerEnabled import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsBalancerEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCleanerChoreEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCleanerChoreEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsNormalizerEnabledRequest; @@ -164,10 +166,14 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRe import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetCleanerChoreRunningRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetCleanerChoreRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; @@ -1931,6 +1937,25 @@ class ConnectionManager { } @Override + public RunCleanerChoreResponse runCleanerChore(RpcController controller, + RunCleanerChoreRequest request) throws ServiceException { + return stub.runCleanerChore(controller, request); + } + + @Override + public SetCleanerChoreRunningResponse setCleanerChoreRunning(RpcController controller, + SetCleanerChoreRunningRequest request) throws ServiceException { + return stub.setCleanerChoreRunning(controller, request); + } + + @Override + public IsCleanerChoreEnabledResponse isCleanerChoreEnabled( + RpcController controller, IsCleanerChoreEnabledRequest request) + throws ServiceException { + return stub.isCleanerChoreEnabled(controller, request); + } + + @Override public CoprocessorServiceResponse execMasterService( RpcController controller, CoprocessorServiceRequest request) throws ServiceException { http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index cbc83dc..5def9a4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2552,6 +2552,39 @@ public class HBaseAdmin implements Admin { }); } + @Override + public boolean setCleanerChoreRunning(final boolean on) throws IOException { + return executeCallable(new MasterCallable<Boolean>(getConnection()) { + @Override + public Boolean call(int callTimeout) throws ServiceException { + return master.setCleanerChoreRunning(null, + RequestConverter.buildSetCleanerChoreRunningRequest(on)).getPrevValue(); + } + }); + } + + @Override + public boolean runCleanerChore() throws IOException { + return executeCallable(new MasterCallable<Boolean>(getConnection()) { + @Override + public Boolean call(int callTimeout) throws ServiceException { + return master.runCleanerChore(null, RequestConverter.buildCleanerChoreRequest()) + .getCleanerChoreRan(); + } + }); + } + + @Override + public boolean isCleanerChoreEnabled() throws IOException { + return executeCallable(new MasterCallable<Boolean>(getConnection()) { + @Override + public Boolean call(int callTimeout) throws ServiceException { + return master.isCleanerChoreEnabled(null, + RequestConverter.buildIsCleanerChoreEnabledRequest()).getValue(); + } + }); + } + private boolean isEncodedRegionName(byte[] regionName) throws IOException { try { HRegionInfo.parseRegionName(regionName); http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java index cac4e97..5e4a163 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java @@ -89,12 +89,14 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableReque import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetCleanerChoreRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsBalancerEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCleanerChoreEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsNormalizerEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest; @@ -104,6 +106,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.NormalizeRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest; @@ -1486,6 +1489,42 @@ public final class RequestConverter { } /** + * @see {@link #buildCleanerChoreRequest} + */ + private static final RunCleanerChoreRequest CLEANER_CHORE_REQUEST = RunCleanerChoreRequest + .newBuilder().build(); + + /** + * Creates a request for running cleaner chore + * @return A {@link RunCleanerChoreRequest} + */ + public static RunCleanerChoreRequest buildCleanerChoreRequest() { + return CLEANER_CHORE_REQUEST; + } + + /** + * Creates a request for enabling/disabling the cleaner chore + * @return A {@link SetCleanerChoreRunningRequest} + */ + public static SetCleanerChoreRunningRequest buildSetCleanerChoreRunningRequest(boolean on) { + return SetCleanerChoreRunningRequest.newBuilder().setOn(on).build(); + } + + /** + * @see {@link #buildIsCleanerChoreEnabledRequest()} + */ + private static final IsCleanerChoreEnabledRequest IS_CLEANER_CHORE_ENABLED_REQUEST = + IsCleanerChoreEnabledRequest.newBuilder().build(); + + /** + * Creates a request for querying the master whether the cleaner chore is enabled + * @return A {@link IsCleanerChoreEnabledRequest} + */ + public static IsCleanerChoreEnabledRequest buildIsCleanerChoreEnabledRequest() { + return IS_CLEANER_CHORE_ENABLED_REQUEST; + } + + /** * Creates a request for querying the master the last flushed sequence Id for a region * @param regionName * @return A {@link GetLastFlushedSequenceIdRequest} http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java index 91d30fd..ba7041e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair; import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreResponse; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.security.access.UserPermission; @@ -303,6 +304,14 @@ public final class ResponseConverter { return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build(); } + /** + * Creates a response for the cleaner chore request + * @return A RunCleanerChoreResponse + */ + public static RunCleanerChoreResponse buildRunCleanerChoreResponse(boolean ran) { + return RunCleanerChoreResponse.newBuilder().setCleanerChoreRan(ran).build(); + } + // End utilities for Admin /**
