Add jmx/nodetool methods to enable/disable hint storage/delivery
Patch by Alexey Zotov and brandonwilliams, reviewed by Alexey Zotov for
CASSANDRA-4750


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9ccaaade
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9ccaaade
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9ccaaade

Branch: refs/heads/cassandra-1.2
Commit: 9ccaaadec636b63f63d3206fc66ef4d035f6a311
Parents: b36a808
Author: Brandon Williams <brandonwilli...@apache.org>
Authored: Wed Jan 9 11:43:15 2013 -0600
Committer: Brandon Williams <brandonwilli...@apache.org>
Committed: Wed Jan 9 11:43:15 2013 -0600

----------------------------------------------------------------------
 CHANGES.txt                                        |    1 +
 .../apache/cassandra/db/HintedHandOffManager.java  |   32 ++++++++++++++-
 .../cassandra/db/HintedHandOffManagerMBean.java    |    3 +
 src/java/org/apache/cassandra/tools/NodeCmd.java   |    8 ++++
 src/java/org/apache/cassandra/tools/NodeProbe.java |   25 +++++++++++
 .../org/apache/cassandra/tools/NodeToolHelp.yaml   |   13 +++++-
 6 files changed, 80 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d4c6697..2346993 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 1.2.1
+ * nodetool methods to enable/disable hint storage/delivery (CASSANDRA-4750)
  * disallow bloom filter false positive chance of 0 (CASSANDRA-5013)
  * add threadpool size adjustment methods to JMXEnabledThreadPoolExecutor and 
    CompactionManagerMBean (CASSANDRA-5044)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java 
b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
index 476af88..8ca0281 100644
--- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
@@ -87,12 +87,15 @@ import org.cliffc.high_scale_lib.NonBlockingHashSet;
 
 public class HintedHandOffManager implements HintedHandOffManagerMBean
 {
+    public static final String MBEAN_NAME = 
"org.apache.cassandra.db:type=HintedHandoffManager";
     public static final HintedHandOffManager instance = new 
HintedHandOffManager();
 
     private static final Logger logger = 
LoggerFactory.getLogger(HintedHandOffManager.class);
     private static final int PAGE_SIZE = 128;
     private static final int LARGE_NUMBER = 65536; // 64k nodes ought to be 
enough for anybody.
 
+    private volatile boolean hintedHandOffPaused = false;
+
     static final CompositeType comparator = 
CompositeType.getInstance(Arrays.<AbstractType<?>>asList(UUIDType.instance, 
Int32Type.instance));
 
     private final NonBlockingHashSet<InetAddress> queuedDeliveries = new 
NonBlockingHashSet<InetAddress>();
@@ -108,7 +111,7 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
         MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
         try
         {
-            mbs.registerMBean(this, new 
ObjectName("org.apache.cassandra.db:type=HintedHandoffManager"));
+            mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
         }
         catch (Exception e)
         {
@@ -256,6 +259,13 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
         if (hintStore.isEmpty())
             return; // nothing to do, don't confuse users by logging a no-op 
handoff
 
+        // check if hints delivery has been paused
+        if (hintedHandOffPaused)
+        {
+            logger.debug("Hints delivery process is paused, aborting");
+            return;
+        }
+
         logger.debug("Checking remote({}) schema before delivering hints", 
endpoint);
         try
         {
@@ -303,6 +313,13 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
 
         while (true)
         {
+            // check if hints delivery has been paused during the process
+            if (hintedHandOffPaused)
+            {
+                logger.debug("Hints delivery process is paused, aborting");
+                break;
+            }
+
             QueryFilter filter = QueryFilter.getSliceFilter(epkey, new 
QueryPath(SystemTable.HINTS_CF), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, 
false, pageSize);
             ColumnFamily hintsPage = 
ColumnFamilyStore.removeDeleted(hintStore.getColumnFamily(filter), 
(int)(System.currentTimeMillis() / 1000));
             if (pagingFinished(hintsPage, startColumn))
@@ -332,6 +349,11 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
                 if (!hint.isLive())
                     continue;
 
+                if (hintedHandOffPaused)
+                {
+                    logger.debug("Hints delivery process is paused, aborting");
+                    break;
+                }
                 startColumn = hint.name();
 
                 ByteBuffer[] components = comparator.split(hint.name());
@@ -384,6 +406,10 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
         }
 
         logger.info(String.format("Finished hinted handoff of %s rows to 
endpoint %s", rowsReplayed, endpoint));
+        if (hintedHandOffPaused)
+        {
+            logger.info("Hints delivery process is paused, not delivering 
further hints");
+        }
     }
 
     /**
@@ -440,6 +466,10 @@ public class HintedHandOffManager implements 
HintedHandOffManagerMBean
         scheduleHintDelivery(InetAddress.getByName(to));
     }
 
+    public void pauseHintsDelivery(boolean b) {
+        hintedHandOffPaused = b;
+    }
+
     public List<String> listEndpointsPendingHints()
     {
         Token.TokenFactory tokenFactory = 
StorageService.getPartitioner().getTokenFactory();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java 
b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java
index 1ba70c4..690a3eb 100644
--- a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java
@@ -45,5 +45,8 @@ public interface HintedHandOffManagerMBean
 
     /** force hint delivery to an endpoint **/
     public void scheduleHintDelivery(String host) throws UnknownHostException;
+
+    /** pause hints delivery process **/
+    public void pauseHintsDelivery(boolean b);
 }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/tools/NodeCmd.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java 
b/src/java/org/apache/cassandra/tools/NodeCmd.java
index d72f3d3..dddc98c 100644
--- a/src/java/org/apache/cassandra/tools/NodeCmd.java
+++ b/src/java/org/apache/cassandra/tools/NodeCmd.java
@@ -102,9 +102,11 @@ public class NodeCmd
         COMPACTIONSTATS,
         DECOMMISSION,
         DISABLEGOSSIP,
+        DISABLEHANDOFF,
         DISABLETHRIFT,
         DRAIN,
         ENABLEGOSSIP,
+        ENABLEHANDOFF,
         ENABLETHRIFT,
         FLUSH,
         GETCOMPACTIONTHRESHOLD,
@@ -118,12 +120,14 @@ public class NodeCmd
         JOIN,
         MOVE,
         NETSTATS,
+        PAUSEHANDOFF,
         PROXYHISTOGRAMS,
         REBUILD,
         REFRESH,
         REMOVETOKEN,
         REMOVENODE,
         REPAIR,
+        RESUMEHANDOFF,
         RING,
         SCRUB,
         SETCACHECAPACITY,
@@ -1006,6 +1010,10 @@ public class NodeCmd
                 case COMPACTIONSTATS : 
nodeCmd.printCompactionStats(System.out); break;
                 case DISABLEGOSSIP   : probe.stopGossiping(); break;
                 case ENABLEGOSSIP    : probe.startGossiping(); break;
+                case DISABLEHANDOFF  : probe.disableHintedHandoff(); break;
+                case ENABLEHANDOFF   : probe.enableHintedHandoff(); break;
+                case PAUSEHANDOFF    : probe.pauseHintsDelivery(); break;
+                case RESUMEHANDOFF   : probe.resumeHintsDelivery(); break;
                 case DISABLETHRIFT   : probe.stopThriftServer(); break;
                 case ENABLETHRIFT    : probe.startThriftServer(); break;
                 case STATUSTHRIFT    : 
nodeCmd.printIsThriftServerRunning(System.out); break;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java 
b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 021a40f..7a7c60c 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -40,6 +40,8 @@ import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutorMBean;
 import org.apache.cassandra.db.ColumnFamilyStoreMBean;
+import org.apache.cassandra.db.HintedHandOffManager;
+import org.apache.cassandra.db.HintedHandOffManagerMBean;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.CompactionManagerMBean;
 import org.apache.cassandra.gms.FailureDetector;
@@ -77,6 +79,7 @@ public class NodeProbe
     private CacheServiceMBean cacheService;
     private PBSPredictorMBean PBSPredictorProxy;
     private StorageProxyMBean spProxy;
+    private HintedHandOffManagerMBean hhProxy;
 
     /**
      * Creates a NodeProbe using the specified JMX host, port, username, and 
password.
@@ -159,6 +162,8 @@ public class NodeProbe
             cacheService = JMX.newMBeanProxy(mbeanServerConn, name, 
CacheServiceMBean.class);
             name = new ObjectName(StorageProxy.MBEAN_NAME);
             spProxy = JMX.newMBeanProxy(mbeanServerConn, name, 
StorageProxyMBean.class);
+            name = new ObjectName(HintedHandOffManager.MBEAN_NAME);
+            hhProxy = JMX.newMBeanProxy(mbeanServerConn, name, 
HintedHandOffManagerMBean.class);
         } catch (MalformedObjectNameException e)
         {
             throw new RuntimeException(
@@ -647,6 +652,26 @@ public class NodeProbe
         return ssProxy.getKeyspaces();
     }
 
+    public void disableHintedHandoff()
+    {
+        spProxy.setHintedHandoffEnabled(false);
+    }
+
+    public void enableHintedHandoff()
+    {
+        spProxy.setHintedHandoffEnabled(true);
+    }
+
+    public void pauseHintsDelivery()
+    {
+        hhProxy.pauseHintsDelivery(true);
+    }
+
+    public void resumeHintsDelivery()
+    {
+        hhProxy.pauseHintsDelivery(false);
+    }
+
     public void stopGossiping()
     {
         ssProxy.stopGossiping();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ccaaade/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml
----------------------------------------------------------------------
diff --git a/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml 
b/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml
index 677b689..2398f01 100644
--- a/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml
+++ b/src/resources/org/apache/cassandra/tools/NodeToolHelp.yaml
@@ -1,4 +1,3 @@
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -50,6 +49,18 @@ commands:
   - name: compactionstats
     help: |
       Print statistics on compactions
+  - name: disablehandoff
+    help: |
+      Disable the future hints storing on the current node
+  - name: enablehandoff
+    help: |
+      Reenable the future hints storing on the current node
+  - name: resumehandoff
+    help: |
+      Resume hints delivery process
+  - name: pausehandoff
+    help: |
+      Pause hints delivery process
   - name: disablegossip
     help: |
       Disable gossip (effectively marking the node dead)

Reply via email to