Author: brandonwilliams Date: Wed Jul 14 16:28:23 2010 New Revision: 964092
URL: http://svn.apache.org/viewvc?rev=964092&view=rev Log: Log thread pool stats when GC is excessive. Patch by brandonwilliams; reviewed by jbellis for CASSANDRA-1275 Modified: cassandra/branches/cassandra-0.6/CHANGES.txt cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/GCInspector.java cassandra/trunk/CHANGES.txt cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java Modified: cassandra/branches/cassandra-0.6/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/CHANGES.txt?rev=964092&r1=964091&r2=964092&view=diff ============================================================================== --- cassandra/branches/cassandra-0.6/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.6/CHANGES.txt Wed Jul 14 16:28:23 2010 @@ -5,6 +5,7 @@ (CASSANDRA-1232) * extend option to lower compaction priority to hinted handoff as well (CASSANDRA-1260) +* log thread pool stats when GC is excessive (CASSANDRA-1275) 0.6.3 Modified: cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/GCInspector.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/GCInspector.java?rev=964092&r1=964091&r2=964092&view=diff ============================================================================== --- cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/GCInspector.java (original) +++ cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/GCInspector.java Wed Jul 14 16:28:23 2010 @@ -27,12 +27,16 @@ import java.util.*; import java.util.concurrent.ExecutorService; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; import org.apache.cassandra.utils.WrappedRunnable; +import org.apache.cassandra.concurrent.IExecutorMBean; +import org.apache.cassandra.db.CompactionManagerMBean; import com.sun.management.GarbageCollectorMXBean; import com.sun.management.GcInfo; import java.lang.management.MemoryUsage; import java.lang.management.ManagementFactory; +import javax.management.JMX; import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; import javax.management.ObjectName; public class GCInspector @@ -42,8 +46,10 @@ public class GCInspector private static final Logger logger = Logger.getLogger(GCInspector.class); final static long INTERVAL_IN_MS = 1000; final static long MIN_DURATION = 200; + final static long MIN_DURATION_TPSTATS = 1000; private HashMap<String, Long> gctimes = new HashMap<String, Long>(); + private final MBeanServer server = ManagementFactory.getPlatformMBeanServer(); List<GarbageCollectorMXBean> beans = new ArrayList<GarbageCollectorMXBean>(); @@ -110,6 +116,35 @@ public class GCInspector logger.info(st); else if (logger.isDebugEnabled()) logger.debug(st); + if (gci.getDuration() > MIN_DURATION_TPSTATS) + { + try + { + logThreadPoolStats(); + } + catch (MalformedObjectNameException e) + { + throw new RuntimeException(e); + } + } + } + } + + private void logThreadPoolStats() throws MalformedObjectNameException + { + ObjectName query = new ObjectName("org.apache.cassandra.concurrent:type=*"); + Iterator<ObjectName> tpiter = server.queryNames(query, null).iterator(); + logger.info(String.format("%-25s%10s%10s", "Pool Name", "Active", "Pending")); + while(tpiter.hasNext()) + { + ObjectName objectName = tpiter.next(); + String poolName = objectName.getKeyProperty("type"); + IExecutorMBean threadPoolProxy = JMX.newMBeanProxy(server, objectName, IExecutorMBean.class); + logger.info(String.format("%-25s%10d%10d", poolName, threadPoolProxy.getActiveCount(), threadPoolProxy.getPendingTasks())); } + // one off for compaction + ObjectName cm = new ObjectName("org.apache.cassandra.db:type=CompactionManager"); + CompactionManagerMBean cmProxy = JMX.newMBeanProxy(server, cm, CompactionManagerMBean.class); + logger.info(String.format("%-25s%10s%10s", "CompactionManager", "n/a", cmProxy.getPendingTasks())); } } Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=964092&r1=964091&r2=964092&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Wed Jul 14 16:28:23 2010 @@ -50,6 +50,7 @@ dev (CASSANDRA-1232) * extend option to lower compaction priority to hinted handoff as well (CASSANDRA-1260) + * log thread pool stats when GC is excessive (CASSANDRA-1275) 0.6.3 Modified: cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java?rev=964092&r1=964091&r2=964092&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java Wed Jul 14 16:28:23 2010 @@ -27,12 +27,16 @@ import java.util.*; import java.util.concurrent.ExecutorService; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; import org.apache.cassandra.utils.WrappedRunnable; +import org.apache.cassandra.concurrent.IExecutorMBean; +import org.apache.cassandra.db.CompactionManagerMBean; import com.sun.management.GarbageCollectorMXBean; import com.sun.management.GcInfo; import java.lang.management.MemoryUsage; import java.lang.management.ManagementFactory; +import javax.management.JMX; import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; import javax.management.ObjectName; public class GCInspector @@ -42,8 +46,10 @@ public class GCInspector private static final Logger logger = LoggerFactory.getLogger(GCInspector.class); final static long INTERVAL_IN_MS = 1000; final static long MIN_DURATION = 200; + final static long MIN_DURATION_TPSTATS = 1000; private HashMap<String, Long> gctimes = new HashMap<String, Long>(); + private final MBeanServer server = ManagementFactory.getPlatformMBeanServer(); List<GarbageCollectorMXBean> beans = new ArrayList<GarbageCollectorMXBean>(); @@ -110,6 +116,35 @@ public class GCInspector logger.info(st); else if (logger.isDebugEnabled()) logger.debug(st); + if (gci.getDuration() > MIN_DURATION_TPSTATS) + { + try + { + logThreadPoolStats(); + } + catch (MalformedObjectNameException e) + { + throw new RuntimeException(e); + } + } + } + } + + private void logThreadPoolStats() throws MalformedObjectNameException + { + ObjectName query = new ObjectName("org.apache.cassandra.concurrent:type=*"); + Iterator<ObjectName> tpiter = server.queryNames(query, null).iterator(); + logger.info(String.format("%-25s%10s%10s", "Pool Name", "Active", "Pending")); + while(tpiter.hasNext()) + { + ObjectName objectName = tpiter.next(); + String poolName = objectName.getKeyProperty("type"); + IExecutorMBean threadPoolProxy = JMX.newMBeanProxy(server, objectName, IExecutorMBean.class); + logger.info(String.format("%-25s%10d%10d", poolName, threadPoolProxy.getActiveCount(), threadPoolProxy.getPendingTasks())); } + // one off for compaction + ObjectName cm = new ObjectName("org.apache.cassandra.db:type=CompactionManager"); + CompactionManagerMBean cmProxy = JMX.newMBeanProxy(server, cm, CompactionManagerMBean.class); + logger.info(String.format("%-25s%10s%10s", "CompactionManager", "n/a", cmProxy.getPendingTasks())); } }