Repository: hbase Updated Branches: refs/heads/branch-1 e9abe0762 -> 67fe516ec
http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-protocol/src/main/protobuf/Master.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/protobuf/Master.proto b/hbase-protocol/src/main/protobuf/Master.proto index b6a56e2..1c60465 100644 --- a/hbase-protocol/src/main/protobuf/Master.proto +++ b/hbase-protocol/src/main/protobuf/Master.proto @@ -345,6 +345,28 @@ message IsCatalogJanitorEnabledResponse { required bool value = 1; } +message RunCleanerChoreRequest { +} + +message RunCleanerChoreResponse { + required bool cleaner_chore_ran = 1; +} + +message SetCleanerChoreRunningRequest { + required bool on = 1; +} + +message SetCleanerChoreRunningResponse { + optional bool prev_value = 1; +} + +message IsCleanerChoreEnabledRequest { +} + +message IsCleanerChoreEnabledResponse { + required bool value = 1; +} + message SnapshotRequest { required SnapshotDescription snapshot = 1; } @@ -699,6 +721,22 @@ service MasterService { rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest) returns(IsCatalogJanitorEnabledResponse); + /** Get a run of the cleaner chore*/ + rpc RunCleanerChore(RunCleanerChoreRequest) + returns(RunCleanerChoreResponse); + + /** + * Enable the cleaner chore on or off. + */ + rpc SetCleanerChoreRunning(SetCleanerChoreRunningRequest) + returns(SetCleanerChoreRunningResponse); + + /** + * Query whether the cleaner chore is enabled. + */ + rpc IsCleanerChoreEnabled(IsCleanerChoreEnabledRequest) + returns(IsCleanerChoreEnabledResponse); + /** * Call a master coprocessor endpoint */ http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 9da081a..2352df2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1053,6 +1053,20 @@ public class HMaster extends HRegionServer implements MasterServices, Server { catalogJanitorChore.getEnabled() : false; } + boolean isCleanerChoreEnabled() { + boolean hfileCleanerFlag = true, logCleanerFlag = true; + + if (hfileCleaner != null) { + hfileCleanerFlag = hfileCleaner.getEnabled(); + } + + if(logCleaner != null) { + logCleanerFlag = logCleaner.getEnabled(); + } + + return (hfileCleanerFlag && logCleanerFlag); + } + private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException { if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) { // In log replay mode, we mark hbase:meta region as recovering in ZK @@ -2713,6 +2727,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return this.hfileCleaner; } + public LogCleaner getLogCleaner() { + return this.logCleaner; + } + /** * @return the underlying snapshot manager */ http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index f51a797..c678c86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -107,6 +107,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsBalancerEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsBalancerEnabledResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCleanerChoreEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCleanerChoreEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsInMaintenanceModeRequest; @@ -149,11 +151,15 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRe import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCleanerChoreResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse.Capability; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetCleanerChoreRunningRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetCleanerChoreRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; @@ -650,6 +656,21 @@ public class MasterRpcServices extends RSRpcServices } @Override + public SetCleanerChoreRunningResponse setCleanerChoreRunning(RpcController c, + SetCleanerChoreRunningRequest req) throws ServiceException { + try { + master.checkInitialized(); + } catch (IOException ioe) { + throw new ServiceException(ioe); + } + boolean prevValue = + master.getLogCleaner().getEnabled() && master.getHFileCleaner().getEnabled(); + master.getLogCleaner().setEnabled(req.getOn()); + master.getHFileCleaner().setEnabled(req.getOn()); + return SetCleanerChoreRunningResponse.newBuilder().setPrevValue(prevValue).build(); + } + + @Override public EnableTableResponse enableTable(RpcController controller, EnableTableRequest request) throws ServiceException { try { @@ -942,6 +963,13 @@ public class MasterRpcServices extends RSRpcServices } @Override + public IsCleanerChoreEnabledResponse isCleanerChoreEnabled(RpcController c, + IsCleanerChoreEnabledRequest req) throws ServiceException { + return IsCleanerChoreEnabledResponse.newBuilder() + .setValue(master.isCleanerChoreEnabled()).build(); + } + + @Override public IsMasterRunningResponse isMasterRunning(RpcController c, IsMasterRunningRequest req) throws ServiceException { try { @@ -1304,6 +1332,19 @@ public class MasterRpcServices extends RSRpcServices } @Override + public RunCleanerChoreResponse runCleanerChore(RpcController c, RunCleanerChoreRequest req) + throws ServiceException { + try { + master.checkInitialized(); + Boolean result = master.getHFileCleaner().runCleaner() + && master.getLogCleaner().runCleaner(); + return ResponseConverter.buildRunCleanerChoreResponse(result); + } catch (IOException ioe) { + throw new ServiceException(ioe); + } + } + + @Override public SetBalancerRunningResponse setBalancerRunning(RpcController c, SetBalancerRunningRequest req) throws ServiceException { try { http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java index 0efcff7..d431b2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.cleaner; import java.io.IOException; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -49,6 +50,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Schedu private final Path oldFileDir; private final Configuration conf; protected List<T> cleanersChain; + private AtomicBoolean enabled = new AtomicBoolean(true); /** * @param name name of the chore being run @@ -119,13 +121,23 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Schedu @Override protected void chore() { + if (getEnabled()) { + runCleaner(); + } else { + LOG.debug("Cleaner disabled! Not cleaning."); + } + } + + public Boolean runCleaner() { try { FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir); checkAndDeleteEntries(files); } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); LOG.warn("Error while cleaning the logs", e); + return false; } + return true; } /** @@ -279,4 +291,15 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Schedu } } } + + /** + * @param enabled + */ + public boolean setEnabled(final boolean enabled) { + return this.enabled.getAndSet(enabled); + } + + public boolean getEnabled() { + return this.enabled.get(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java index 0bd0da5..7cf9492 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java @@ -288,6 +288,72 @@ public class TestCleanerChore { Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class)); } + @Test + public void testDeleteFileWithCleanerEnabled() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // Enable cleaner + chore.setEnabled(true); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path child = new Path(parent, "child"); + Path file = new Path(child, "someFile"); + fs.mkdirs(child); + + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertFalse("File didn't get deleted", fs.exists(file)); + assertFalse("Empty directory didn't get deleted", fs.exists(child)); + assertFalse("Empty directory didn't get deleted", fs.exists(parent)); + } + + @Test + public void testDeleteFileWithCleanerDisabled() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // Disable cleaner + chore.setEnabled(false); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path child = new Path(parent, "child"); + Path file = new Path(child, "someFile"); + fs.mkdirs(child); + + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertTrue("File got deleted with cleaner disabled", fs.exists(file)); + assertTrue("Directory got deleted", fs.exists(child)); + assertTrue("Directory got deleted", fs.exists(parent)); + } + private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> { public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs, http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-shell/src/main/ruby/hbase/admin.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 50d1e7d..b5050b0 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -211,6 +211,26 @@ module Hbase end #---------------------------------------------------------------------------------------------- + # Request cleaner chore (for garbage collection of HFiles and WAL files) + def cleaner_chore_run() + @admin.runCleanerChore() + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable the cleaner chore + # Returns previous cleaner chore switch setting. + def cleaner_chore_switch(enableDisable) + @admin.setCleanerChoreRunning(java.lang.Boolean::valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query on the cleaner chore state (enabled/disabled?) + # Returns cleaner chore state (true signifies enabled). + def cleaner_chore_enabled() + @admin.isCleanerChoreEnabled() + end + + #---------------------------------------------------------------------------------------------- # Enables a table def enable(table_name) tableExists(table_name) http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index f94e334..9576cc7 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -335,6 +335,9 @@ Shell.load_command_group( catalogjanitor_run catalogjanitor_switch catalogjanitor_enabled + cleaner_chore_run + cleaner_chore_switch + cleaner_chore_enabled compact_rs compaction_state trace http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb new file mode 100644 index 0000000..cd78cc5 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreEnabled < Command + def help + return <<-EOF +Query for the Cleaner chore state (enabled/disabled?). +Examples: + + hbase> cleaner_chore_enabled +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.cleaner_chore_enabled()? "true" : "false" + ]) + end + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb new file mode 100644 index 0000000..eb14966 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreRun < Command + def help + return <<-EOF +Cleaner command for garbage collection of HFiles and WAL files. + + hbase> cleaner_chore_run + +EOF + end + def command() + format_simple_command do + admin.cleaner_chore_run() + end + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/67fe516e/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb new file mode 100644 index 0000000..78c1cc1 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CleanerChoreSwitch < Command + def help + return <<-EOF +Enable/Disable Cleaner chore. Returns previous Cleaner chore state. +Examples: + + hbase> cleaner_chore_switch true + hbase> cleaner_chore_switch false +EOF + end + + def command(enableDisable) + format_simple_command do + formatter.row([ + admin.cleaner_chore_switch(enableDisable)? "true" : "false" + ]) + end + end + end + end +end
