Repository: cassandra Updated Branches: refs/heads/trunk b207f0d95 -> 1096f9f5e
Make gc_log and gc_warn settable at runtime Patch by Edward Capriolo; reviewed by Jon Haddad for CASSANDRA-12661 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1096f9f5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1096f9f5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1096f9f5 Branch: refs/heads/trunk Commit: 1096f9f5e77ff7b17cc7f9fe5aba008834899251 Parents: b207f0d Author: Edward Capriolo <edlinuxg...@gmail.com> Authored: Sat Sep 17 12:07:20 2016 -0400 Committer: Blake Eggleston <bdeggles...@gmail.com> Committed: Tue Apr 11 17:14:11 2017 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + conf/cassandra.yaml | 14 ++-- .../org/apache/cassandra/config/Config.java | 2 +- .../apache/cassandra/service/GCInspector.java | 75 ++++++++++++++++---- .../cassandra/service/GCInspectorMXBean.java | 11 ++- .../cassandra/service/GCInspectorTest.java | 60 ++++++++++++++++ 6 files changed, 139 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5c38307..4f3cb3b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661) * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354) * Skip building views during base table streams on range movements (CASSANDRA-13065) * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197) http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index f2c4c84..1c54830 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -981,10 +981,6 @@ inter_dc_tcp_nodelay: false tracetype_query_ttl: 86400 tracetype_repair_ttl: 604800 -# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level -# This threshold can be adjusted to minimize logging if necessary -# gc_log_threshold_in_ms: 200 - # If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at # INFO level # UDFs (user defined functions) are disabled by default. @@ -1062,10 +1058,14 @@ unlogged_batch_across_partitions_warn_threshold: 10 # Log a warning when compacting partitions larger than this value compaction_large_partition_warning_threshold_mb: 100 +# GC Pauses greater than 200 ms will be logged at INFO level +# This threshold can be adjusted to minimize logging if necessary +# gc_log_threshold_in_ms: 200 + # GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level -# Adjust the threshold based on your application throughput requirement -# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level -gc_warn_threshold_in_ms: 1000 +# Adjust the threshold based on your application throughput requirement. Setting to 0 +# will deactivate the feature. +# gc_warn_threshold_in_ms: 1000 # Maximum size of any value in SSTables. Safety measure to detect SSTable corruption # early. Any value size larger than this threshold will result into marking an SSTable http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 1461cd4..b86429c 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -267,7 +267,7 @@ public class Config public volatile int index_summary_resize_interval_in_minutes = 60; public int gc_log_threshold_in_ms = 200; - public int gc_warn_threshold_in_ms = 0; + public int gc_warn_threshold_in_ms = 1000; // TTL for different types of trace events. public int tracetype_query_ttl = (int) TimeUnit.DAYS.toSeconds(1); http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/src/java/org/apache/cassandra/service/GCInspector.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java index e7cfcd0..b016249 100644 --- a/src/java/org/apache/cassandra/service/GCInspector.java +++ b/src/java/org/apache/cassandra/service/GCInspector.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.service; +import java.io.IOException; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryUsage; @@ -27,7 +28,11 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import javax.management.InstanceAlreadyExistsException; +import javax.management.MBeanRegistrationException; import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.NotCompliantMBeanException; import javax.management.Notification; import javax.management.NotificationListener; import javax.management.ObjectName; @@ -48,9 +53,8 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean { public static final String MBEAN_NAME = "org.apache.cassandra.service:type=GCInspector"; private static final Logger logger = LoggerFactory.getLogger(GCInspector.class); - final static long MIN_LOG_DURATION = DatabaseDescriptor.getGCLogThreshold(); - final static long GC_WARN_THRESHOLD_IN_MS = DatabaseDescriptor.getGCWarnThreshold(); - final static long STAT_THRESHOLD = GC_WARN_THRESHOLD_IN_MS != 0 ? GC_WARN_THRESHOLD_IN_MS : MIN_LOG_DURATION; + private volatile static long gcLogThreshholdInMs = DatabaseDescriptor.getGCLogThreshold(); + private volatile static long gcWarnThreasholdInMs = DatabaseDescriptor.getGCWarnThreshold(); /* * The field from java.nio.Bits that tracks the total number of allocated @@ -58,6 +62,7 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean */ final static Field BITS_TOTAL_CAPACITY; + static { Field temp = null; @@ -146,10 +151,12 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean GarbageCollectorMXBean gc = ManagementFactory.newPlatformMXBeanProxy(mbs, name.getCanonicalName(), GarbageCollectorMXBean.class); gcStates.put(gc.getName(), new GCState(gc, assumeGCIsPartiallyConcurrent(gc), assumeGCIsOldGen(gc))); } - - mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); + ObjectName me = new ObjectName(MBEAN_NAME); + if (!mbs.isRegistered(me)) + mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); } - catch (Exception e) + catch (RuntimeException | InstanceAlreadyExistsException | MBeanRegistrationException | + NotCompliantMBeanException | MalformedObjectNameException | IOException e) { throw new RuntimeException(e); } @@ -276,16 +283,15 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean if (state.compareAndSet(prev, new State(duration, bytes, prev))) break; } - - String st = sb.toString(); - if (GC_WARN_THRESHOLD_IN_MS != 0 && duration > GC_WARN_THRESHOLD_IN_MS) - logger.warn(st); - else if (duration > MIN_LOG_DURATION) - logger.info(st); + + if (gcWarnThreasholdInMs != 0 && duration > gcWarnThreasholdInMs) + logger.warn(sb.toString()); + else if (duration > gcLogThreshholdInMs) + logger.info(sb.toString()); else if (logger.isTraceEnabled()) - logger.trace(st); + logger.trace(sb.toString()); - if (duration > STAT_THRESHOLD) + if (duration > this.getStatusThresholdInMs()) StatusLogger.log(); // if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure @@ -328,4 +334,45 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean return -1; } } + + @Override + public void setGcWarnThresholdInMs(long threshold) + { + if (threshold < 0) + throw new IllegalArgumentException("Threashold must be greater than 0"); + if (threshold != 0 && threshold <= gcLogThreshholdInMs) + throw new IllegalArgumentException("Threashold must be greater than gcLogTreasholdInMs which is currently " + + gcLogThreshholdInMs); + gcWarnThreasholdInMs = threshold; + } + + @Override + public long getGcWarnThresholdInMs() + { + return gcWarnThreasholdInMs; + } + + @Override + public void setGcLogThresholdInMs(long threshold) + { + if (threshold <= 0) + throw new IllegalArgumentException("Threashold must be greater than 0"); + if (gcWarnThreasholdInMs != 0 && threshold > gcWarnThreasholdInMs) + throw new IllegalArgumentException("Threashold must be less than gcWarnTreasholdInMs which is currently " + + gcWarnThreasholdInMs); + gcLogThreshholdInMs = threshold; + } + + @Override + public long getGcLogThresholdInMs() + { + return gcLogThreshholdInMs; + } + + @Override + public long getStatusThresholdInMs() + { + return gcWarnThreasholdInMs != 0 ? gcWarnThreasholdInMs : gcLogThreshholdInMs; + } + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/src/java/org/apache/cassandra/service/GCInspectorMXBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/GCInspectorMXBean.java b/src/java/org/apache/cassandra/service/GCInspectorMXBean.java index c26a67c..08795ed 100644 --- a/src/java/org/apache/cassandra/service/GCInspectorMXBean.java +++ b/src/java/org/apache/cassandra/service/GCInspectorMXBean.java @@ -20,6 +20,13 @@ package org.apache.cassandra.service; public interface GCInspectorMXBean { - // returns { interval (ms), max(gc real time (ms)), sum(gc real time (ms)), sum((gc real time (ms))^2), sum(gc bytes), count(gc) } - public double[] getAndResetStats(); + /* @returns { interval (ms), max(gc real time (ms)), sum(gc real time (ms)), sum((gc real time (ms))^2), sum(gc bytes), count(gc) } + * + */ + double[] getAndResetStats(); + void setGcWarnThresholdInMs(long threshold); + long getGcWarnThresholdInMs(); + void setGcLogThresholdInMs(long threshold); + long getGcLogThresholdInMs(); + long getStatusThresholdInMs(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/1096f9f5/test/unit/org/apache/cassandra/service/GCInspectorTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/service/GCInspectorTest.java b/test/unit/org/apache/cassandra/service/GCInspectorTest.java new file mode 100644 index 0000000..872c736 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/GCInspectorTest.java @@ -0,0 +1,60 @@ +package org.apache.cassandra.service; + +import org.apache.cassandra.OrderedJUnit4ClassRunner; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(OrderedJUnit4ClassRunner.class) +public class GCInspectorTest +{ + + GCInspector gcInspector; + + @BeforeClass + public static void setupDatabaseDescriptor() + { + DatabaseDescriptor.daemonInitialization(); + } + + @Before + public void before(){ + gcInspector = new GCInspector(); + } + + @Test + public void ensureStaticFieldsHydrateFromConfig() + { + Assert.assertEquals(DatabaseDescriptor.getGCLogThreshold(), gcInspector.getGcLogThresholdInMs()); + Assert.assertEquals(DatabaseDescriptor.getGCWarnThreshold(), gcInspector.getGcWarnThresholdInMs()); + } + + @Test + public void ensureStatusIsCalculated() + { + Assert.assertTrue(gcInspector.getStatusThresholdInMs() > 0); + } + + @Test(expected=IllegalArgumentException.class) + public void ensureWarnGreaterThanLog() + { + gcInspector.setGcWarnThresholdInMs(gcInspector.getGcLogThresholdInMs()); + } + + @Test + public void ensureZeroIsOk() + { + gcInspector.setGcWarnThresholdInMs(0); + Assert.assertEquals(gcInspector.getStatusThresholdInMs(), gcInspector.getGcLogThresholdInMs()); + } + + @Test(expected=IllegalArgumentException.class) + public void ensureLogLessThanWarn() + { + gcInspector.setGcLogThresholdInMs(gcInspector.getGcWarnThresholdInMs() + 1); + } + +}
