Add jmx/nodetool methods to enable/disable hint storage/delivery Patch by Alexey Zotov and brandonwilliams, reviewed by Alexey Zotov for CASSANDRA-4750
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9ccaaade Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9ccaaade Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9ccaaade Branch: refs/heads/cassandra-1.2 Commit: 9ccaaadec636b63f63d3206fc66ef4d035f6a311 Parents: b36a808 Author: Brandon Williams <brandonwilli...@apache.org> Authored: Wed Jan 9 11:43:15 2013 -0600 Committer: Brandon Williams <brandonwilli...@apache.org> Committed: Wed Jan 9 11:43:15 2013 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/HintedHandOffManager.java | 32 ++++++++++++++- .../cassandra/db/HintedHandOffManagerMBean.java | 3 + src/java/org/apache/cassandra/tools/NodeCmd.java | 8 ++++ src/java/org/apache/cassandra/tools/NodeProbe.java | 25 +++++++++++ .../org/apache/cassandra/tools/NodeToolHelp.yaml | 13 +++++- 6 files changed, 80 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d4c6697..2346993 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.2.1 + * nodetool methods to enable/disable hint storage/delivery (CASSANDRA-4750) * disallow bloom filter false positive chance of 0 (CASSANDRA-5013) * add threadpool size adjustment methods to JMXEnabledThreadPoolExecutor and CompactionManagerMBean (CASSANDRA-5044) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/db/HintedHandOffManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index 476af88..8ca0281 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -87,12 +87,15 @@ import org.cliffc.high_scale_lib.NonBlockingHashSet; public class HintedHandOffManager implements HintedHandOffManagerMBean { + public static final String MBEAN_NAME = "org.apache.cassandra.db:type=HintedHandoffManager"; public static final HintedHandOffManager instance = new HintedHandOffManager(); private static final Logger logger = LoggerFactory.getLogger(HintedHandOffManager.class); private static final int PAGE_SIZE = 128; private static final int LARGE_NUMBER = 65536; // 64k nodes ought to be enough for anybody. + private volatile boolean hintedHandOffPaused = false; + static final CompositeType comparator = CompositeType.getInstance(Arrays.<AbstractType<?>>asList(UUIDType.instance, Int32Type.instance)); private final NonBlockingHashSet<InetAddress> queuedDeliveries = new NonBlockingHashSet<InetAddress>(); @@ -108,7 +111,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); try { - mbs.registerMBean(this, new ObjectName("org.apache.cassandra.db:type=HintedHandoffManager")); + mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); } catch (Exception e) { @@ -256,6 +259,13 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean if (hintStore.isEmpty()) return; // nothing to do, don't confuse users by logging a no-op handoff + // check if hints delivery has been paused + if (hintedHandOffPaused) + { + logger.debug("Hints delivery process is paused, aborting"); + return; + } + logger.debug("Checking remote({}) schema before delivering hints", endpoint); try { @@ -303,6 +313,13 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean while (true) { + // check if hints delivery has been paused during the process + if (hintedHandOffPaused) + { + logger.debug("Hints delivery process is paused, aborting"); + break; + } + QueryFilter filter = QueryFilter.getSliceFilter(epkey, new QueryPath(SystemTable.HINTS_CF), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, pageSize); ColumnFamily hintsPage = ColumnFamilyStore.removeDeleted(hintStore.getColumnFamily(filter), (int)(System.currentTimeMillis() / 1000)); if (pagingFinished(hintsPage, startColumn)) @@ -332,6 +349,11 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean if (!hint.isLive()) continue; + if (hintedHandOffPaused) + { + logger.debug("Hints delivery process is paused, aborting"); + break; + } startColumn = hint.name(); ByteBuffer[] components = comparator.split(hint.name()); @@ -384,6 +406,10 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean } logger.info(String.format("Finished hinted handoff of %s rows to endpoint %s", rowsReplayed, endpoint)); + if (hintedHandOffPaused) + { + logger.info("Hints delivery process is paused, not delivering further hints"); + } } /** @@ -440,6 +466,10 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean scheduleHintDelivery(InetAddress.getByName(to)); } + public void pauseHintsDelivery(boolean b) { + hintedHandOffPaused = b; + } + public List<String> listEndpointsPendingHints() { Token.TokenFactory tokenFactory = StorageService.getPartitioner().getTokenFactory(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java index 1ba70c4..690a3eb 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java @@ -45,5 +45,8 @@ public interface HintedHandOffManagerMBean /** force hint delivery to an endpoint **/ public void scheduleHintDelivery(String host) throws UnknownHostException; + + /** pause hints delivery process **/ + public void pauseHintsDelivery(boolean b); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/tools/NodeCmd.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index d72f3d3..dddc98c 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -102,9 +102,11 @@ public class NodeCmd COMPACTIONSTATS, DECOMMISSION, DISABLEGOSSIP, + DISABLEHANDOFF, DISABLETHRIFT, DRAIN, ENABLEGOSSIP, + ENABLEHANDOFF, ENABLETHRIFT, FLUSH, GETCOMPACTIONTHRESHOLD, @@ -118,12 +120,14 @@ public class NodeCmd JOIN, MOVE, NETSTATS, + PAUSEHANDOFF, PROXYHISTOGRAMS, REBUILD, REFRESH, REMOVETOKEN, REMOVENODE, REPAIR, + RESUMEHANDOFF, RING, SCRUB, SETCACHECAPACITY, @@ -1006,6 +1010,10 @@ public class NodeCmd case COMPACTIONSTATS : nodeCmd.printCompactionStats(System.out); break; case DISABLEGOSSIP : probe.stopGossiping(); break; case ENABLEGOSSIP : probe.startGossiping(); break; + case DISABLEHANDOFF : probe.disableHintedHandoff(); break; + case ENABLEHANDOFF : probe.enableHintedHandoff(); break; + case PAUSEHANDOFF : probe.pauseHintsDelivery(); break; + case RESUMEHANDOFF : probe.resumeHintsDelivery(); break; case DISABLETHRIFT : probe.stopThriftServer(); break; case ENABLETHRIFT : probe.startThriftServer(); break; case STATUSTHRIFT : nodeCmd.printIsThriftServerRunning(System.out); break; http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 021a40f..7a7c60c 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -40,6 +40,8 @@ import com.google.common.collect.Iterables; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutorMBean; import org.apache.cassandra.db.ColumnFamilyStoreMBean; +import org.apache.cassandra.db.HintedHandOffManager; +import org.apache.cassandra.db.HintedHandOffManagerMBean; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionManagerMBean; import org.apache.cassandra.gms.FailureDetector; @@ -77,6 +79,7 @@ public class NodeProbe private CacheServiceMBean cacheService; private PBSPredictorMBean PBSPredictorProxy; private StorageProxyMBean spProxy; + private HintedHandOffManagerMBean hhProxy; /** * Creates a NodeProbe using the specified JMX host, port, username, and password. @@ -159,6 +162,8 @@ public class NodeProbe cacheService = JMX.newMBeanProxy(mbeanServerConn, name, CacheServiceMBean.class); name = new ObjectName(StorageProxy.MBEAN_NAME); spProxy = JMX.newMBeanProxy(mbeanServerConn, name, StorageProxyMBean.class); + name = new ObjectName(HintedHandOffManager.MBEAN_NAME); + hhProxy = JMX.newMBeanProxy(mbeanServerConn, name, HintedHandOffManagerMBean.class); } catch (MalformedObjectNameException e) { throw new RuntimeException( @@ -647,6 +652,26 @@ public class NodeProbe return ssProxy.getKeyspaces(); } + public void disableHintedHandoff() + { + spProxy.setHintedHandoffEnabled(false); + } + + public void enableHintedHandoff() + { + spProxy.setHintedHandoffEnabled(true); + } + + public void pauseHintsDelivery() + { + hhProxy.pauseHintsDelivery(true); + } + + public void resumeHintsDelivery() + { + hhProxy.pauseHintsDelivery(false); + } + public void stopGossiping() { ssProxy.stopGossiping(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml ---------------------------------------------------------------------- diff --git a/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml b/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml index 677b689..2398f01 100644 --- a/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml +++ b/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml @@ -1,4 +1,3 @@ - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -50,6 +49,18 @@ commands: - name: compactionstats help: | Print statistics on compactions + - name: disablehandoff + help: | + Disable the future hints storing on the current node + - name: enablehandoff + help: | + Reenable the future hints storing on the current node + - name: resumehandoff + help: | + Resume hints delivery process + - name: pausehandoff + help: | + Pause hints delivery process - name: disablegossip help: | Disable gossip (effectively marking the node dead)
