Repository: cassandra Updated Branches: refs/heads/trunk 3513fbcfb -> 9581209b3
Make prepared statement cache size configurable patch by Robert Stupp; reviewed by Benjamin Lerer for CASSANDRA-11555 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9581209b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9581209b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9581209b Branch: refs/heads/trunk Commit: 9581209b35922a0758c6bd158d4336a17cfe86aa Parents: 3513fbc Author: Robert Stupp <sn...@snazy.de> Authored: Mon May 2 17:03:56 2016 +0200 Committer: Robert Stupp <sn...@snazy.de> Committed: Mon May 2 17:03:56 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + conf/cassandra.yaml | 28 ++++++++ .../org/apache/cassandra/config/Config.java | 11 +++ .../cassandra/config/DatabaseDescriptor.java | 45 ++++++++++++ .../apache/cassandra/cql3/QueryProcessor.java | 72 +++++++++++--------- 5 files changed, 125 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c49249c..c802031 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.6 + * Make prepared statement cache size configurable (CASSANDRA-11555) * Integrated JMX authentication and authorization (CASSANDRA-10091) * Add units to stress ouput (CASSANDRA-11352) * Fix PER PARTITION LIMIT for single and multi partitions queries (CASSANDRA-11603) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 48bad2c..9eb55e1 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -215,6 +215,34 @@ disk_failure_policy: stop # ignore: ignore fatal errors and let the batches fail commit_failure_policy: stop +# Maximum size of the native protocol prepared statement cache +# +# Valid values are either "auto" (omitting the value) or a value greater 0. +# +# Note that specifying a too large value will result in long running GCs and possbily +# out-of-memory errors. Keep the value at a small fraction of the heap. +# +# If you constantly see "prepared statements discarded in the last minute because +# cache limit reached" messages, the first step is to investigate the root cause +# of these messages and check whether prepared statements are used correctly - +# i.e. use bind markers for variable parts. +# +# Do only change the default value, if you really have more prepared statements than +# fit in the cache. In most cases it is not neccessary to change this value. +# Constantly re-preparing statements is a performance penalty. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +prepared_statements_cache_size_mb: + +# Maximum size of the Thrift prepared statement cache +# +# If you do not use Thrift at all, it is safe to leave this value at "auto". +# +# See description of 'prepared_statements_cache_size_mb' above for more information. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +thrift_prepared_statements_cache_size_mb: + # Maximum size of the key cache in memory. # # Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 809966d..02635bf 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -288,6 +288,17 @@ public class Config public int windows_timer_interval = 0; + /** + * Size of the CQL prepared statements cache in MB. + * Defaults to 1/256th of the heap size or 10MB, whichever is greater. + */ + public Long prepared_statements_cache_size_mb = null; + /** + * Size of the Thrift prepared statements cache in MB. + * Defaults to 1/256th of the heap size or 10MB, whichever is greater. + */ + public Long thrift_prepared_statements_cache_size_mb = null; + public boolean enable_user_defined_functions = false; public boolean enable_scripted_user_defined_functions = false; /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index b98d103..d8acdb8 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -96,6 +96,9 @@ public class DatabaseDescriptor private static RequestSchedulerId requestSchedulerId; private static RequestSchedulerOptions requestSchedulerOptions; + private static long preparedStatementsCacheSizeInMB; + private static long thriftPreparedStatementsCacheSizeInMB; + private static long keyCacheSizeInMB; private static long counterCacheSizeInMB; private static long indexSummaryCapacityInMB; @@ -641,6 +644,38 @@ public class DatabaseDescriptor try { + // if prepared_statements_cache_size_mb option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MB), 10MB)" + preparedStatementsCacheSizeInMB = (conf.prepared_statements_cache_size_mb == null) + ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256)) + : conf.prepared_statements_cache_size_mb; + + if (preparedStatementsCacheSizeInMB <= 0) + throw new NumberFormatException(); // to escape duplicating error message + } + catch (NumberFormatException e) + { + throw new ConfigurationException("prepared_statements_cache_size_mb option was set incorrectly to '" + + conf.prepared_statements_cache_size_mb + "', supported values are <integer> >= 0.", false); + } + + try + { + // if thrift_prepared_statements_cache_size_mb option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MB), 10MB)" + thriftPreparedStatementsCacheSizeInMB = (conf.thrift_prepared_statements_cache_size_mb == null) + ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256)) + : conf.thrift_prepared_statements_cache_size_mb; + + if (thriftPreparedStatementsCacheSizeInMB <= 0) + throw new NumberFormatException(); // to escape duplicating error message + } + catch (NumberFormatException e) + { + throw new ConfigurationException("thrift_prepared_statements_cache_size_mb option was set incorrectly to '" + + conf.thrift_prepared_statements_cache_size_mb + "', supported values are <integer> >= 0.", false); + } + + try + { // if key_cache_size_in_mb option was set to "auto" then size of the cache should be "min(5% of Heap (in MB), 100MB) keyCacheSizeInMB = (conf.key_cache_size_in_mb == null) ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)), 100) @@ -1992,6 +2027,16 @@ public class DatabaseDescriptor return conf.windows_timer_interval; } + public static long getPreparedStatementsCacheSizeMB() + { + return preparedStatementsCacheSizeInMB; + } + + public static long getThriftPreparedStatementsCacheSizeMB() + { + return thriftPreparedStatementsCacheSizeInMB; + } + public static boolean enableUserDefinedFunctions() { return conf.enable_user_defined_functions; http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/cql3/QueryProcessor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index 5f4b0f6..39a9c03 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -34,9 +34,9 @@ import org.slf4j.LoggerFactory; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import com.googlecode.concurrentlinkedhashmap.EntryWeigher; -import com.googlecode.concurrentlinkedhashmap.EvictionListener; import org.antlr.runtime.*; import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.functions.FunctionName; @@ -65,7 +65,6 @@ public class QueryProcessor implements QueryHandler private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class); private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_BEST).ignoreKnownSingletons(); - private static final long MAX_CACHE_PREPARED_MEMORY = Runtime.getRuntime().maxMemory() / 256; private static final EntryWeigher<MD5Digest, ParsedStatement.Prepared> cqlMemoryUsageWeigher = new EntryWeigher<MD5Digest, ParsedStatement.Prepared>() { @@ -97,45 +96,48 @@ public class QueryProcessor implements QueryHandler public static final CQLMetrics metrics = new CQLMetrics(); private static final AtomicInteger lastMinuteEvictionsCount = new AtomicInteger(0); + private static final AtomicInteger thriftLastMinuteEvictionsCount = new AtomicInteger(0); static { preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, ParsedStatement.Prepared>() - .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY) + .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB())) .weigher(cqlMemoryUsageWeigher) - .listener(new EvictionListener<MD5Digest, ParsedStatement.Prepared>() - { - public void onEviction(MD5Digest md5Digest, ParsedStatement.Prepared prepared) - { - metrics.preparedStatementsEvicted.inc(); - lastMinuteEvictionsCount.incrementAndGet(); - } + .listener((md5Digest, prepared) -> { + metrics.preparedStatementsEvicted.inc(); + lastMinuteEvictionsCount.incrementAndGet(); }).build(); thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, ParsedStatement.Prepared>() - .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY) + .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB())) .weigher(thriftMemoryUsageWeigher) - .listener(new EvictionListener<Integer, ParsedStatement.Prepared>() - { - public void onEviction(Integer integer, ParsedStatement.Prepared prepared) - { - metrics.preparedStatementsEvicted.inc(); - lastMinuteEvictionsCount.incrementAndGet(); - } + .listener((integer, prepared) -> { + metrics.preparedStatementsEvicted.inc(); + thriftLastMinuteEvictionsCount.incrementAndGet(); }) .build(); - ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(new Runnable() - { - public void run() - { - long count = lastMinuteEvictionsCount.getAndSet(0); - if (count > 0) - logger.info("{} prepared statements discarded in the last minute because cache limit reached ({})", - count, - FBUtilities.prettyPrintMemory(MAX_CACHE_PREPARED_MEMORY)); - } + ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(() -> { + long count = lastMinuteEvictionsCount.getAndSet(0); + if (count > 0) + logger.warn("{} prepared statements discarded in the last minute because cache limit reached ({} MB)", + count, + DatabaseDescriptor.getPreparedStatementsCacheSizeMB()); + count = thriftLastMinuteEvictionsCount.getAndSet(0); + if (count > 0) + logger.warn("{} prepared Thrift statements discarded in the last minute because cache limit reached ({} MB)", + count, + DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()); }, 1, 1, TimeUnit.MINUTES); + + logger.info("Initialized prepared statement caches with {} MB (native) and {} MB (Thrift)", + DatabaseDescriptor.getPreparedStatementsCacheSizeMB(), + DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()); + } + + private static long capacityToBytes(long cacheSizeMB) + { + return cacheSizeMB * 1024 * 1024; } public static int preparedStatementsCount() @@ -428,18 +430,24 @@ public class QueryProcessor implements QueryHandler // (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine. long statementSize = measure(prepared.statement); // don't execute the statement if it's bigger than the allowed threshold - if (statementSize > MAX_CACHE_PREPARED_MEMORY) - throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d bytes.", - statementSize, - MAX_CACHE_PREPARED_MEMORY)); if (forThrift) { + if (statementSize > capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB())) + throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...", + statementSize, + DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB(), + queryString.substring(0, 200))); Integer statementId = computeThriftId(queryString, keyspace); thriftPreparedStatements.put(statementId, prepared); return ResultMessage.Prepared.forThrift(statementId, prepared.boundNames); } else { + if (statementSize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB())) + throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...", + statementSize, + DatabaseDescriptor.getPreparedStatementsCacheSizeMB(), + queryString.substring(0, 200))); MD5Digest statementId = computeId(queryString, keyspace); preparedStatements.put(statementId, prepared); return new ResultMessage.Prepared(statementId, prepared);