http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 2617c85..0e654b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -26,7 +26,6 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -57,7 +56,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.MetaMigrationConvertingToPB; @@ -65,7 +63,6 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.PleaseHoldException; -import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -110,7 +107,6 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; @@ -126,7 +122,6 @@ import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.EncryptionTest; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HBaseFsckRepair; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.Pair; @@ -380,7 +375,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { " is not set - not publishing status"); } else { clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass); - Threads.setDaemonThreadRunning(clusterStatusPublisherChore.getThread()); + getChoreService().scheduleChore(clusterStatusPublisherChore); } } activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this); @@ -719,11 +714,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // been assigned. status.setStatus("Starting balancer and catalog janitor"); this.clusterStatusChore = new ClusterStatusChore(this, balancer); - Threads.setDaemonThreadRunning(clusterStatusChore.getThread()); + getChoreService().scheduleChore(clusterStatusChore); this.balancerChore = new BalancerChore(this); - Threads.setDaemonThreadRunning(balancerChore.getThread()); + getChoreService().scheduleChore(balancerChore); this.catalogJanitorChore = new CatalogJanitor(this, this); - Threads.setDaemonThreadRunning(catalogJanitorChore.getThread()); + getChoreService().scheduleChore(catalogJanitorChore); status.setStatus("Starting namespace manager"); initNamespace(); @@ -1032,16 +1027,13 @@ public class HMaster extends HRegionServer implements MasterServices, Server { new LogCleaner(cleanerInterval, this, conf, getMasterFileSystem().getFileSystem(), getMasterFileSystem().getOldLogDir()); - Threads.setDaemonThreadRunning(logCleaner.getThread(), - getServerName().toShortString() + ".oldLogCleaner"); + getChoreService().scheduleChore(logCleaner); //start the hfile archive cleaner thread Path archiveDir = HFileArchiveUtil.getArchivePath(conf); this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem() .getFileSystem(), archiveDir); - Threads.setDaemonThreadRunning(hfileCleaner.getThread(), - getServerName().toShortString() + ".archivedHFileCleaner"); - + getChoreService().scheduleChore(hfileCleaner); serviceStarted = true; if (LOG.isTraceEnabled()) { LOG.trace("Started service threads"); @@ -1070,8 +1062,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { LOG.debug("Stopping service threads"); } // Clean up and close up shop - if (this.logCleaner!= null) this.logCleaner.interrupt(); - if (this.hfileCleaner != null) this.hfileCleaner.interrupt(); + if (this.logCleaner != null) this.logCleaner.cancel(true); + if (this.hfileCleaner != null) this.hfileCleaner.cancel(true); if (this.activeMasterManager != null) this.activeMasterManager.stop(); if (this.serverManager != null) this.serverManager.stop(); if (this.assignmentManager != null) this.assignmentManager.stop(); @@ -1081,16 +1073,16 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private void stopChores() { if (this.balancerChore != null) { - this.balancerChore.interrupt(); + this.balancerChore.cancel(true); } if (this.clusterStatusChore != null) { - this.clusterStatusChore.interrupt(); + this.clusterStatusChore.cancel(true); } if (this.catalogJanitorChore != null) { - this.catalogJanitorChore.interrupt(); + this.catalogJanitorChore.cancel(true); } if (this.clusterStatusPublisherChore != null){ - clusterStatusPublisherChore.interrupt(); + clusterStatusPublisherChore.cancel(true); } }
http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index 23ef6a5..29969d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -39,29 +39,30 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SplitLogCounters; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination; import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; -import org.apache.hadoop.hbase.wal.DefaultWALProvider; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.wal.DefaultWALProvider; +import org.apache.hadoop.hbase.wal.WALFactory; import com.google.common.annotations.VisibleForTesting; @@ -102,6 +103,7 @@ public class SplitLogManager { private final Stoppable stopper; private final Configuration conf; + private final ChoreService choreService; public static final int DEFAULT_UNASSIGNED_TIMEOUT = (3 * 60 * 1000); // 3 min @@ -138,6 +140,7 @@ public class SplitLogManager { this.server = server; this.conf = conf; this.stopper = stopper; + this.choreService = new ChoreService(serverName.toString() + "_splitLogManager_"); if (server.getCoordinatedStateManager() != null) { SplitLogManagerCoordination coordination = ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) @@ -154,8 +157,7 @@ public class SplitLogManager { this.timeoutMonitor = new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper); - Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName - + ".splitLogManagerTimeoutMonitor"); + choreService.scheduleChore(timeoutMonitor); } private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException { @@ -528,8 +530,11 @@ public class SplitLogManager { } public void stop() { + if (choreService != null) { + choreService.shutdown(); + } if (timeoutMonitor != null) { - timeoutMonitor.interrupt(); + timeoutMonitor.cancel(true); } } @@ -683,11 +688,11 @@ public class SplitLogManager { /** * Periodically checks all active tasks and resubmits the ones that have timed out */ - private class TimeoutMonitor extends Chore { + private class TimeoutMonitor extends ScheduledChore { private long lastLog = 0; public TimeoutMonitor(final int period, Stoppable stopper) { - super("SplitLogManager Timeout Monitor", period, stopper); + super("SplitLogManager Timeout Monitor", stopper, period); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java index 21fe272..bbbfdf2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerChore.java @@ -18,28 +18,27 @@ package org.apache.hadoop.hbase.master.balancer; +import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.master.HMaster; -import java.io.IOException; - /** * Chore that will call HMaster.balance{@link org.apache.hadoop.hbase.master.HMaster#balance()} when * needed. */ @InterfaceAudience.Private -public class BalancerChore extends Chore { +public class BalancerChore extends ScheduledChore { private static final Log LOG = LogFactory.getLog(BalancerChore.class); private final HMaster master; public BalancerChore(HMaster master) { - super(master.getServerName() + "-BalancerChore", - master.getConfiguration().getInt("hbase.balancer.period", 300000), - master); + super(master.getServerName() + "-BalancerChore", master, master.getConfiguration().getInt( + "hbase.balancer.period", 300000)); this.master = master; } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java index 046dfb7..58e5808 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterStatusChore.java @@ -18,28 +18,27 @@ package org.apache.hadoop.hbase.master.balancer; +import java.io.InterruptedIOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.LoadBalancer; -import java.io.InterruptedIOException; - /** * Chore that will feed the balancer the cluster status. */ @InterfaceAudience.Private -public class ClusterStatusChore extends Chore { +public class ClusterStatusChore extends ScheduledChore { private static final Log LOG = LogFactory.getLog(ClusterStatusChore.class); private final HMaster master; private final LoadBalancer balancer; public ClusterStatusChore(HMaster master, LoadBalancer balancer) { - super(master.getServerName() + "-ClusterStatusChore", - master.getConfiguration().getInt("hbase.balancer.statusPeriod", 60000), - master); + super(master.getServerName() + "-ClusterStatusChore", master, master.getConfiguration().getInt( + "hbase.balancer.statusPeriod", 60000)); this.master = master; this.balancer = balancer; } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java index 6e2f4fd..cc4cc01 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java @@ -27,8 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.util.FSUtils; @@ -41,7 +41,7 @@ import com.google.common.collect.Lists; * Abstract Cleaner that uses a chain of delegates to clean a directory of files * @param <T> Cleaner delegate class that is dynamically loaded from configuration */ -public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore { +public abstract class CleanerChore<T extends FileCleanerDelegate> extends ScheduledChore { private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName()); @@ -61,7 +61,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore */ public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf, FileSystem fs, Path oldFileDir, String confKey) { - super(name, sleepPeriod, s); + super(name, s, sleepPeriod); this.fs = fs; this.oldFileDir = oldFileDir; this.conf = conf; @@ -224,7 +224,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore Iterable<FileStatus> deletableValidFiles = validFiles; // check each of the cleaners for the valid files for (T cleaner : cleanersChain) { - if (cleaner.isStopped() || this.stopper.isStopped()) { + if (cleaner.isStopped() || getStopper().isStopped()) { LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any more files in:" + this.oldFileDir); return false; http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index a76c9c6..6f8b9fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -58,7 +58,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; @@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.HealthCheckChore; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableDescriptors; @@ -128,10 +129,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Repor import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler; import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler; -import org.apache.hadoop.hbase.wal.DefaultWALProvider; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.trace.SpanReceiverHost; @@ -149,6 +147,9 @@ import org.apache.hadoop.hbase.util.JvmPauseMonitor; import org.apache.hadoop.hbase.util.Sleeper; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; +import org.apache.hadoop.hbase.wal.DefaultWALProvider; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; @@ -320,15 +321,20 @@ public class HRegionServer extends HasThread implements MetricsRegionServer metricsRegionServer; private SpanReceiverHost spanReceiverHost; + /** + * ChoreService used to schedule tasks that we want to run periodically + */ + private final ChoreService choreService; + /* * Check for compactions requests. */ - Chore compactionChecker; + ScheduledChore compactionChecker; /* * Check for flushes */ - Chore periodicFlusher; + ScheduledChore periodicFlusher; protected volatile WALFactory walFactory; @@ -367,7 +373,7 @@ public class HRegionServer extends HasThread implements private HealthCheckChore healthCheckChore; /** The nonce manager chore. */ - private Chore nonceManagerChore; + private ScheduledChore nonceManagerChore; private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap(); @@ -550,6 +556,7 @@ public class HRegionServer extends HasThread implements rpcServices.start(); putUpWebUI(); this.walRoller = new LogRoller(this, this); + this.choreService = new ChoreService(getServerName().toString()); } protected void login(UserProvider user, String host) throws IOException { @@ -777,8 +784,8 @@ public class HRegionServer extends HasThread implements movedRegionsCleaner = MovedRegionsCleaner.createAndStart(this); if (this.nonceManager != null) { - // Create the chore that cleans up nonces. - nonceManagerChore = this.nonceManager.createCleanupChore(this); + // Create the scheduled chore that cleans up nonces. + nonceManagerChore = this.nonceManager.createCleanupScheduledChore(this); } // Setup RPC client for master communication @@ -927,17 +934,10 @@ public class HRegionServer extends HasThread implements if(this.hMemManager != null) this.hMemManager.stop(); if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary(); if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary(); - if (this.compactionChecker != null) - this.compactionChecker.interrupt(); - if (this.healthCheckChore != null) { - this.healthCheckChore.interrupt(); - } - if (this.nonceManagerChore != null) { - this.nonceManagerChore.interrupt(); - } - if (this.storefileRefresher != null) { - this.storefileRefresher.interrupt(); - } + if (this.compactionChecker != null) this.compactionChecker.cancel(true); + if (this.healthCheckChore != null) this.healthCheckChore.cancel(true); + if (this.nonceManagerChore != null) this.nonceManagerChore.cancel(true); + if (this.storefileRefresher != null) this.storefileRefresher.cancel(true); // Stop the snapshot and other procedure handlers, forcefully killing all running tasks if (rspmHost != null) { @@ -1293,7 +1293,7 @@ public class HRegionServer extends HasThread implements private void startHeapMemoryManager() { this.hMemManager = HeapMemoryManager.create(this.conf, this.cacheFlusher, this); if (this.hMemManager != null) { - this.hMemManager.start(); + this.hMemManager.start(getChoreService()); } } @@ -1408,7 +1408,7 @@ public class HRegionServer extends HasThread implements /* * Inner class that runs on a long period checking if regions need compaction. */ - private static class CompactionChecker extends Chore { + private static class CompactionChecker extends ScheduledChore { private final HRegionServer instance; private final int majorCompactPriority; private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE; @@ -1416,7 +1416,7 @@ public class HRegionServer extends HasThread implements CompactionChecker(final HRegionServer h, final int sleepTime, final Stoppable stopper) { - super("CompactionChecker", sleepTime, h); + super("CompactionChecker", stopper, sleepTime); this.instance = h; LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime)); @@ -1462,12 +1462,12 @@ public class HRegionServer extends HasThread implements } } - class PeriodicMemstoreFlusher extends Chore { + class PeriodicMemstoreFlusher extends ScheduledChore { final HRegionServer server; final static int RANGE_OF_DELAY = 20000; //millisec final static int MIN_DELAY_TIME = 3000; //millisec public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) { - super(server.getServerName() + "-MemstoreFlusherChore", cacheFlushInterval, server); + super(server.getServerName() + "-MemstoreFlusherChore", server, cacheFlushInterval); this.server = server; } @@ -1608,22 +1608,12 @@ public class HRegionServer extends HasThread implements Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + ".logRoller", uncaughtExceptionHandler); this.cacheFlusher.start(uncaughtExceptionHandler); - Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), getName() + - ".compactionChecker", uncaughtExceptionHandler); - Threads.setDaemonThreadRunning(this.periodicFlusher.getThread(), getName() + - ".periodicFlusher", uncaughtExceptionHandler); - if (this.healthCheckChore != null) { - Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), getName() + ".healthChecker", - uncaughtExceptionHandler); - } - if (this.nonceManagerChore != null) { - Threads.setDaemonThreadRunning(this.nonceManagerChore.getThread(), getName() + ".nonceCleaner", - uncaughtExceptionHandler); - } - if (this.storefileRefresher != null) { - Threads.setDaemonThreadRunning(this.storefileRefresher.getThread(), getName() + ".storefileRefresher", - uncaughtExceptionHandler); - } + + if (this.compactionChecker != null) choreService.scheduleChore(compactionChecker); + if (this.periodicFlusher != null) choreService.scheduleChore(periodicFlusher); + if (this.healthCheckChore != null) choreService.scheduleChore(healthCheckChore); + if (this.nonceManagerChore != null) choreService.scheduleChore(nonceManagerChore); + if (this.storefileRefresher != null) choreService.scheduleChore(storefileRefresher); // Leases is not a Thread. Internally it runs a daemon thread. If it gets // an unhandled exception, it will just exit. @@ -1716,8 +1706,8 @@ public class HRegionServer extends HasThread implements // Verify that all threads are alive if (!(leases.isAlive() && cacheFlusher.isAlive() && walRoller.isAlive() - && this.compactionChecker.isAlive() - && this.periodicFlusher.isAlive())) { + && this.compactionChecker.isScheduled() + && this.periodicFlusher.isScheduled())) { stop("One or more threads are no longer alive -- stop"); return false; } @@ -1962,21 +1952,18 @@ public class HRegionServer extends HasThread implements * have already been called. */ protected void stopServiceThreads() { - if (this.nonceManagerChore != null) { - Threads.shutdown(this.nonceManagerChore.getThread()); - } - if (this.compactionChecker != null) { - Threads.shutdown(this.compactionChecker.getThread()); - } - if (this.periodicFlusher != null) { - Threads.shutdown(this.periodicFlusher.getThread()); - } + // clean up the scheduled chores + if (this.choreService != null) choreService.shutdown(); + if (this.nonceManagerChore != null) nonceManagerChore.cancel(true); + if (this.compactionChecker != null) compactionChecker.cancel(true); + if (this.periodicFlusher != null) periodicFlusher.cancel(true); + if (this.healthCheckChore != null) healthCheckChore.cancel(true); + if (this.storefileRefresher != null) storefileRefresher.cancel(true); + if (this.cacheFlusher != null) { this.cacheFlusher.join(); } - if (this.healthCheckChore != null) { - Threads.shutdown(this.healthCheckChore.getThread()); - } + if (this.spanReceiverHost != null) { this.spanReceiverHost.closeReceivers(); } @@ -2002,9 +1989,6 @@ public class HRegionServer extends HasThread implements this.replicationSinkHandler.stopReplicationService(); } } - if (this.storefileRefresher != null) { - Threads.shutdown(this.storefileRefresher.getThread()); - } } /** @@ -2403,6 +2387,11 @@ public class HRegionServer extends HasThread implements return service; } + @Override + public ChoreService getChoreService() { + return choreService; + } + // // Main program and support routines // @@ -2904,13 +2893,13 @@ public class HRegionServer extends HasThread implements /** * Creates a Chore thread to clean the moved region cache. */ - protected static class MovedRegionsCleaner extends Chore implements Stoppable { + protected static class MovedRegionsCleaner extends ScheduledChore implements Stoppable { private HRegionServer regionServer; Stoppable stoppable; private MovedRegionsCleaner( HRegionServer regionServer, Stoppable stoppable){ - super("MovedRegionsCleaner for region "+regionServer, TIMEOUT_REGION_MOVED, stoppable); + super("MovedRegionsCleaner for region " + regionServer, stoppable, TIMEOUT_REGION_MOVED); this.regionServer = regionServer; this.stoppable = stoppable; } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index b44c84c..2aa5eba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -26,16 +26,16 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; -import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.util.ReflectionUtils; import com.google.common.annotations.VisibleForTesting; @@ -183,11 +183,11 @@ public class HeapMemoryManager { return true; } - public void start() { - LOG.info("Starting HeapMemoryTuner chore."); - this.heapMemTunerChore = new HeapMemoryTunerChore(); - Threads.setDaemonThreadRunning(heapMemTunerChore.getThread()); - if (tunerOn) { + public void start(ChoreService service) { + LOG.info("Starting HeapMemoryTuner chore."); + this.heapMemTunerChore = new HeapMemoryTunerChore(); + service.scheduleChore(heapMemTunerChore); + if (tunerOn) { // Register HeapMemoryTuner as a memstore flush listener memStoreFlusher.registerFlushRequestListener(heapMemTunerChore); } @@ -196,7 +196,8 @@ public class HeapMemoryManager { public void stop() { // The thread is Daemon. Just interrupting the ongoing process. LOG.info("Stoping HeapMemoryTuner chore."); - this.heapMemTunerChore.interrupt(); + this.heapMemTunerChore.cancel(true); + } // Used by the test cases. @@ -211,7 +212,7 @@ public class HeapMemoryManager { return this.heapOccupancyPercent; } - private class HeapMemoryTunerChore extends Chore implements FlushRequestListener { + private class HeapMemoryTunerChore extends ScheduledChore implements FlushRequestListener { private HeapMemoryTuner heapMemTuner; private AtomicLong blockedFlushCount = new AtomicLong(); private AtomicLong unblockedFlushCount = new AtomicLong(); @@ -220,28 +221,13 @@ public class HeapMemoryManager { private boolean alarming = false; public HeapMemoryTunerChore() { - super(server.getServerName() + "-HeapMemoryTunerChore", defaultChorePeriod, server); + super(server.getServerName() + "-HeapMemoryTunerChore", server, defaultChorePeriod); Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass( HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class); heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration()); } @Override - protected void sleep() { - if (!alarming) { - super.sleep(); - } else { - // we are in the alarm state, so sleep only for a short fixed period - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // Interrupted, propagate - Thread.currentThread().interrupt(); - } - } - } - - @Override protected void chore() { // Sample heap occupancy MemoryUsage memUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); @@ -254,6 +240,16 @@ public class HeapMemoryManager { " is above heap occupancy alarm watermark (" + heapOccupancyLowWatermark + ")"); alarming = true; } + + triggerNow(); + try { + // Need to sleep ourselves since we've told the chore's sleeper + // to skip the next sleep cycle. + Thread.sleep(1000); + } catch (InterruptedException e) { + // Interrupted, propagate + Thread.currentThread().interrupt(); + } } else { if (alarming) { LOG.info("heapOccupancyPercent " + heapOccupancyPercent + http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index 3af7129..d839bed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -18,15 +18,13 @@ */ package org.apache.hadoop.hbase.regionserver; -import com.google.protobuf.Service; - import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentMap; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.master.TableLockManager; @@ -34,6 +32,8 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Regio import org.apache.hadoop.hbase.wal.WAL; import org.apache.zookeeper.KeeperException; +import com.google.protobuf.Service; + /** * Services provided by {@link HRegionServer} */ http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java index 128cb6f..0d974b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java @@ -18,20 +18,19 @@ */ package org.apache.hadoop.hbase.regionserver; -import java.util.Date; import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import com.google.common.annotations.VisibleForTesting; @@ -247,13 +246,13 @@ public class ServerNonceManager { } /** - * Creates a chore that is used to clean up old nonces. + * Creates a scheduled chore that is used to clean up old nonces. * @param stoppable Stoppable for the chore. - * @return Chore; the chore is not started. + * @return ScheduledChore; the scheduled chore is not started. */ - public Chore createCleanupChore(Stoppable stoppable) { + public ScheduledChore createCleanupScheduledChore(Stoppable stoppable) { // By default, it will run every 6 minutes (30 / 5). - return new Chore("nonceCleaner", deleteNonceGracePeriod / 5, stoppable) { + return new ScheduledChore("nonceCleaner", stoppable, deleteNonceGracePeriod / 5) { @Override protected void chore() { cleanUpOldNonces(); http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java index f3c7e6a..4918391 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java @@ -25,9 +25,9 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils; @@ -42,7 +42,7 @@ import org.apache.hadoop.util.StringUtils; * primary region). */ @InterfaceAudience.Private -public class StorefileRefresherChore extends Chore { +public class StorefileRefresherChore extends ScheduledChore { private static final Log LOG = LogFactory.getLog(StorefileRefresherChore.class); @@ -69,7 +69,7 @@ public class StorefileRefresherChore extends Chore { public StorefileRefresherChore(int period, boolean onlyMetaRefresh, HRegionServer regionServer, Stoppable stoppable) { - super("StorefileRefresherChore", period, stoppable); + super("StorefileRefresherChore", stoppable, period); this.period = period; this.regionServer = regionServer; this.hfileTtl = this.regionServer.getConfiguration().getLong( http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java index 997692f..4b997b4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; @@ -181,5 +182,10 @@ public class ReplicationSyncUp extends Configured implements Tool { public ClusterConnection getConnection() { return null; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 23673b6..309a1c2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -36,12 +36,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.AuthUtil; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotEnabledException; @@ -773,8 +775,15 @@ public final class Canary implements Tool { public static void main(String[] args) throws Exception { final Configuration conf = HBaseConfiguration.create(); - AuthUtil.launchAuthChore(conf); + final ChoreService choreService = new ChoreService("CANARY_TOOL"); + final ScheduledChore authChore = AuthUtil.getAuthChore(conf); + if (authChore != null) { + choreService.scheduleChore(authChore); + } + int exitCode = ToolRunner.run(conf, new Canary(), args); + + choreService.shutdown(); System.exit(exitCode); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java index fc4e74c..21714af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java @@ -23,10 +23,11 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; @@ -53,6 +54,7 @@ public class ConnectionCache { private final UserGroupInformation realUser; private final UserProvider userProvider; private final Configuration conf; + private final ChoreService choreService; private final ThreadLocal<String> effectiveUserNames = new ThreadLocal<String>() { @@ -69,8 +71,8 @@ public class ConnectionCache { @Override public void stop(String why) { isStopped = true;} @Override public boolean isStopped() {return isStopped;} }; - - Chore cleaner = new Chore("ConnectionCleaner", cleanInterval, stoppable) { + this.choreService = new ChoreService("ConnectionCache"); + ScheduledChore cleaner = new ScheduledChore("ConnectionCleaner", stoppable, cleanInterval) { @Override protected void chore() { for (Map.Entry<String, ConnectionInfo> entry: connections.entrySet()) { @@ -93,7 +95,7 @@ public class ConnectionCache { } }; // Start the daemon cleaner chore - Threads.setDaemonThreadRunning(cleaner.getThread()); + choreService.scheduleChore(cleaner); this.realUser = userProvider.getCurrent().getUGI(); this.realUserName = realUser.getShortUserName(); this.userProvider = userProvider; @@ -115,6 +117,13 @@ public class ConnectionCache { } /** + * Called when cache is no longer needed so that it can perform cleanup operations + */ + public void shutdown() { + if (choreService != null) choreService.shutdown(); + } + + /** * Caller doesn't close the admin afterwards. * We need to manage it and close it properly. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java index dcb5001..b70a1ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java @@ -44,8 +44,8 @@ import org.apache.hadoop.hbase.regionserver.Leases; import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.ServerNonceManager; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; @@ -229,6 +229,11 @@ public class MockRegionServerServices implements RegionServerServices { } @Override + public ChoreService getChoreService() { + return null; + } + + @Override public void updateRegionFavoredNodesMapping(String encodedRegionName, List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) { } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index ae78a3e..87936e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -33,9 +33,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil; @@ -76,7 +77,7 @@ public class TestHFileArchiving { UTIL.startMiniCluster(); // We don't want the cleaner to remove files. The tests do that. - UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt(); + UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().cancel(true); } private static void setupConf(Configuration conf) { @@ -350,6 +351,7 @@ public class TestHFileArchiving { @Test public void testCleaningRace() throws Exception { final long TEST_TIME = 20 * 1000; + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration(); Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace"); @@ -368,7 +370,7 @@ public class TestHFileArchiving { // The cleaner should be looping without long pauses to reproduce the race condition. HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir); try { - cleaner.start(); + choreService.scheduleChore(cleaner); // Keep creating/archiving new files while the cleaner is running in the other thread long startTime = System.currentTimeMillis(); @@ -403,7 +405,8 @@ public class TestHFileArchiving { } } finally { stoppable.stop("test end"); - cleaner.join(); + cleaner.cancel(true); + choreService.shutdown(); fs.delete(rootDir, true); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java index 52b0f40..a78eafa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -211,6 +212,7 @@ public class TestZooKeeperTableArchiveClient { Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); @@ -249,7 +251,7 @@ public class TestZooKeeperTableArchiveClient { // need to be checked) in 'otherTable' and the files (which should be retained) in the 'table' CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size() + 3); // run the cleaner - cleaner.start(); + choreService.scheduleChore(cleaner); // wait for the cleaner to check all the files finished.await(); // stop the cleaner @@ -411,8 +413,9 @@ public class TestZooKeeperTableArchiveClient { */ private void runCleaner(HFileCleaner cleaner, CountDownLatch finished, Stoppable stop) throws InterruptedException { + final ChoreService choreService = new ChoreService("CLEANER_SERVER_NAME"); // run the cleaner - cleaner.start(); + choreService.scheduleChore(cleaner); // wait for the cleaner to check all the files finished.await(); // stop the cleaner http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java index 49be5c8..a839860 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java @@ -33,10 +33,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; @@ -44,13 +43,14 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.security.SecureBulkLoadUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.HFileTestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.apache.hadoop.hbase.security.SecureBulkLoadUtil; /** * Test cases for the "load" half of the HFileOutputFormat bulk load @@ -97,7 +97,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads * HFiles that fit snugly inside those regions */ - @Test + @Test(timeout = 60000) public void testSimpleLoad() throws Exception { runTest("testSimpleLoad", BloomType.NONE, new byte[][][] { @@ -110,7 +110,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads * HFiles that cross the boundaries of those regions */ - @Test + @Test(timeout = 60000) public void testRegionCrossingLoad() throws Exception { runTest("testRegionCrossingLoad", BloomType.NONE, new byte[][][] { @@ -122,7 +122,7 @@ public class TestLoadIncrementalHFiles { /** * Test loading into a column family that has a ROW bloom filter. */ - @Test + @Test(timeout = 60000) public void testRegionCrossingRowBloom() throws Exception { runTest("testRegionCrossingLoadRowBloom", BloomType.ROW, new byte[][][] { @@ -134,7 +134,7 @@ public class TestLoadIncrementalHFiles { /** * Test loading into a column family that has a ROWCOL bloom filter. */ - @Test + @Test(timeout = 60000) public void testRegionCrossingRowColBloom() throws Exception { runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL, new byte[][][] { @@ -147,7 +147,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads HFiles that have * different region boundaries than the table pre-split. */ - @Test + @Test(timeout = 60000) public void testSimpleHFileSplit() throws Exception { runTest("testHFileSplit", BloomType.NONE, new byte[][] { @@ -165,7 +165,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads HFiles that cross the boundaries * and have different region boundaries than the table pre-split. */ - @Test + @Test(timeout = 60000) public void testRegionCrossingHFileSplit() throws Exception { testRegionCrossingHFileSplit(BloomType.NONE); } @@ -174,7 +174,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads HFiles that cross the boundaries * have a ROW bloom filter and a different region boundaries than the table pre-split. */ - @Test + @Test(timeout = 60000) public void testRegionCrossingHFileSplitRowBloom() throws Exception { testRegionCrossingHFileSplit(BloomType.ROW); } @@ -183,7 +183,7 @@ public class TestLoadIncrementalHFiles { * Test case that creates some regions and loads HFiles that cross the boundaries * have a ROWCOL bloom filter and a different region boundaries than the table pre-split. */ - @Test + @Test(timeout = 60000) public void testRegionCrossingHFileSplitRowColBloom() throws Exception { testRegionCrossingHFileSplit(BloomType.ROWCOL); } @@ -277,7 +277,7 @@ public class TestLoadIncrementalHFiles { /** * Test loading into a column family that does not exist. */ - @Test + @Test(timeout = 60000) public void testNonexistentColumnFamilyLoad() throws Exception { String testName = "testNonexistentColumnFamilyLoad"; byte[][][] hFileRanges = new byte[][][] { @@ -306,7 +306,7 @@ public class TestLoadIncrementalHFiles { } } - @Test + @Test(timeout = 60000) public void testSplitStoreFile() throws IOException { Path dir = util.getDataTestDirOnTestFS("testSplitHFile"); FileSystem fs = util.getTestFileSystem(); @@ -410,7 +410,7 @@ public class TestLoadIncrementalHFiles { } } - @Test + @Test(timeout = 60000) public void testLoadTooMayHFiles() throws Exception { Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles"); FileSystem fs = util.getTestFileSystem(); @@ -444,7 +444,7 @@ public class TestLoadIncrementalHFiles { loader.run(args); } - @Test + @Test(timeout = 60000) public void testTableWithCFNameStartWithUnderScore() throws Exception { Path dir = util.getDataTestDirOnTestFS("cfNameStartWithUnderScore"); FileSystem fs = util.getTestFileSystem(); http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 5dae8ce..4bc3cba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; @@ -98,8 +99,8 @@ import org.apache.hadoop.hbase.regionserver.Leases; import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.ServerNonceManager; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; @@ -539,6 +540,11 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { } @Override + public ChoreService getChoreService() { + return null; + } + + @Override public void updateRegionFavoredNodesMapping(String encodedRegionName, List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) { } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java index 077b68b..406e532 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java @@ -28,6 +28,7 @@ import java.util.concurrent.Semaphore; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -319,5 +320,10 @@ public class TestActiveMasterManager { public ActiveMasterManager getActiveMasterManager() { return activeMasterManager; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index e4c1dbb..b88c747 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -195,6 +196,11 @@ public class TestCatalogJanitor { @Override public void stop(String why) { } + + @Override + public ChoreService getChoreService() { + return null; + } } /** @@ -231,6 +237,11 @@ public class TestCatalogJanitor { } @Override + public ChoreService getChoreService() { + return null; + } + + @Override public MasterFileSystem getMasterFileSystem() { return this.mfs; } @@ -613,7 +624,7 @@ public class TestCatalogJanitor { assertTrue(janitor.cleanParent(parent, regions.get(parent))); services.stop("test finished"); - janitor.join(); + janitor.cancel(true); } /** @@ -681,7 +692,7 @@ public class TestCatalogJanitor { assertEquals(2, janitor.scan()); services.stop("test finished"); - janitor.join(); + janitor.cancel(true); } /** @@ -844,7 +855,7 @@ public class TestCatalogJanitor { FSUtils.delete(fs, rootdir, true); services.stop("Test finished"); server.stop("Test finished"); - janitor.join(); + janitor.cancel(true); } /** @@ -929,7 +940,7 @@ public class TestCatalogJanitor { // cleanup services.stop("Test finished"); server.stop("shutdown"); - janitor.join(); + janitor.cancel(true); } private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java index 56e86dc..d9ec036 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -92,6 +93,11 @@ public class TestClockSkewDetection { @Override public void stop(String why) { } + + @Override + public ChoreService getChoreService() { + return null; + } }, null, false); LOG.debug("regionServerStartup 1"); http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java index f6a7953..c065a59 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java @@ -46,6 +46,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -161,6 +162,11 @@ public class TestSplitLogManager { public MetaTableLocator getMetaTableLocator() { return null; } + + @Override + public ChoreService getChoreService() { + return null; + } } static Stoppable stopper = new Stoppable() { http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java index 2aba875..13e4c16 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java @@ -37,15 +37,16 @@ import java.util.concurrent.Future; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; @@ -344,9 +345,10 @@ public class TestTableLockManager { int familyValues = admin.getTableDescriptor(tableName).getFamily(family).getValues().size(); StoppableImplementation stopper = new StoppableImplementation(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); //alter table every 10 sec - Chore alterThread = new Chore("Alter Chore", 10000, stopper) { + ScheduledChore alterThread = new ScheduledChore("Alter Chore", stopper, 10000) { @Override protected void chore() { Random random = new Random(); @@ -365,7 +367,7 @@ public class TestTableLockManager { }; //split table every 5 sec - Chore splitThread = new Chore("Split thread", 5000, stopper) { + ScheduledChore splitThread = new ScheduledChore("Split thread", stopper, 5000) { @Override public void chore() { try { @@ -390,8 +392,8 @@ public class TestTableLockManager { } }; - alterThread.start(); - splitThread.start(); + choreService.scheduleChore(alterThread); + choreService.scheduleChore(splitThread); TEST_UTIL.waitTableEnabled(tableName); while (true) { List<HRegionInfo> regions = admin.getTableRegions(tableName); @@ -421,6 +423,7 @@ public class TestTableLockManager { } admin.close(); + choreService.shutdown(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java index d1d26ed..4665593 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -156,7 +157,6 @@ public class TestHFileCleaner { LOG.debug("Kept hfiles: " + file.getPath().getName()); } - cleaner.interrupt(); // reset the edge back to the original edge EnvironmentEdgeManager.injectEdge(originalEdge); } @@ -247,5 +247,10 @@ public class TestHFileCleaner { public boolean isStopped() { return false; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java index 72ce7b1..a872679 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java @@ -27,15 +27,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; @@ -120,8 +121,6 @@ public class TestHFileLinkCleaner { } assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); - - cleaner.interrupt(); } private static Path getFamilyDirPath (final Path rootDir, final TableName table, @@ -181,5 +180,10 @@ public class TestHFileLinkCleaner { public boolean isStopped() { return false; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java index d7f29b9..eea520c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -192,5 +193,10 @@ public class TestLogsCleaner { public boolean isStopped() { return false; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 179ae9a..d35c23e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -34,15 +34,12 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.NotServingRegionException; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; @@ -57,13 +54,15 @@ import org.apache.hadoop.hbase.client.MetaScanner; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; +import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.Pair; @@ -215,8 +214,9 @@ public class TestEndToEndSplitTransaction { Stoppable stopper = new StoppableImplementation(); RegionSplitter regionSplitter = new RegionSplitter(table); RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME); + final ChoreService choreService = new ChoreService("TEST_SERVER"); - regionChecker.start(); + choreService.scheduleChore(regionChecker); regionSplitter.start(); //wait until the splitter is finished @@ -315,17 +315,16 @@ public class TestEndToEndSplitTransaction { /** * Checks regions using MetaScanner, MetaTableAccessor and HTable methods */ - static class RegionChecker extends Chore { + static class RegionChecker extends ScheduledChore { Connection connection; Configuration conf; TableName tableName; Throwable ex; RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) throws IOException { - super("RegionChecker", 10, stopper); + super("RegionChecker", stopper, 10); this.conf = conf; this.tableName = tableName; - this.setDaemon(true); this.connection = ConnectionFactory.createConnection(conf); } @@ -410,7 +409,7 @@ public class TestEndToEndSplitTransaction { verify(); } catch (Throwable ex) { this.ex = ex; - stopper.stop("caught exception"); + getStopper().stop("caught exception"); } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index e69e735..8ad7d87 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -27,6 +27,7 @@ import java.lang.management.ManagementFactory; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; @@ -110,7 +111,8 @@ public class TestHeapMemoryManager { new RegionServerStub(conf)); long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); memStoreFlusher.flushType = FlushType.ABOVE_HIGHER_MARK; memStoreFlusher.requestFlush(null, false); memStoreFlusher.requestFlush(null, false); @@ -150,7 +152,8 @@ public class TestHeapMemoryManager { new RegionServerStub(conf)); long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); blockCache.evictBlock(null); blockCache.evictBlock(null); blockCache.evictBlock(null); @@ -185,7 +188,8 @@ public class TestHeapMemoryManager { // Let the system start with default values for memstore heap and block cache size. HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf)); - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner CustomHeapMemoryTuner.memstoreSize = 0.78f; CustomHeapMemoryTuner.blockCacheSize = 0.02f; @@ -214,7 +218,8 @@ public class TestHeapMemoryManager { HeapMemoryTuner.class); HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf)); - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); CustomHeapMemoryTuner.memstoreSize = 0.78f; CustomHeapMemoryTuner.blockCacheSize = 0.02f; Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up @@ -240,7 +245,8 @@ public class TestHeapMemoryManager { new RegionServerStub(conf)); long oldMemstoreSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); CustomHeapMemoryTuner.memstoreSize = 0.7f; CustomHeapMemoryTuner.blockCacheSize = 0.3f; Thread.sleep(1500); @@ -282,7 +288,8 @@ public class TestHeapMemoryManager { conf)); long oldMemstoreSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; - heapMemoryManager.start(); + final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); + heapMemoryManager.start(choreService); CustomHeapMemoryTuner.memstoreSize = 0.4f; CustomHeapMemoryTuner.blockCacheSize = 0.4f; Thread.sleep(1500); @@ -489,6 +496,11 @@ public class TestHeapMemoryManager { public ServerName getServerName() { return ServerName.valueOf("server1",4000,12345); } + + @Override + public ChoreService getChoreService() { + return null; + } } static class CustomHeapMemoryTuner implements HeapMemoryTuner { http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerNonceManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerNonceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerNonceManager.java index 9a97df2..940f715 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerNonceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerNonceManager.java @@ -19,16 +19,19 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.HConstants.NO_NONCE; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; import org.apache.hadoop.hbase.util.Threads; @@ -86,7 +89,7 @@ public class TestServerNonceManager { EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); - Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class)); + ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class)); edge.setValue(1); assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); @@ -119,7 +122,7 @@ public class TestServerNonceManager { EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); - Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class)); + ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class)); // Add nonces from WAL, including dups. edge.setValue(12); nm.reportOperationFromWal(NO_NONCE, 1, 8); http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java index 84d3ea8..48be76b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -136,6 +137,11 @@ public class TestSplitLogWorker { public MetaTableLocator getMetaTableLocator() { return null; } + + @Override + public ChoreService getChoreService() { + return null; + } } private void waitForCounter(AtomicLong ctr, long oldval, long newval, long timems) http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java index a07b708..e0172d6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java @@ -26,6 +26,7 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClusterId; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -183,5 +184,10 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic { public boolean isStopped() { return this.isStopped; } + + @Override + public ChoreService getChoreService() { + return null; + } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/af84b746/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java index 2e3fe08..a5df432 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClusterId; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -286,5 +287,10 @@ public class TestReplicationTrackerZKImpl { public boolean isStopped() { return this.isStopped; } + + @Override + public ChoreService getChoreService() { + return null; + } } }
