Repository: cassandra Updated Branches: refs/heads/cassandra-2.1 e25d94e6e -> 681c380b5
Metrics for prepared stmt usage and eviction Patch by Robbie Strickland; review by Tyler Hobbs for CASSANDRA-7930 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/19c6cc19 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/19c6cc19 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/19c6cc19 Branch: refs/heads/cassandra-2.1 Commit: 19c6cc1982d2146a99ccaf6dccc087fe88d5785f Parents: 169ec3d Author: Robbie Strickland <rostrickl...@gmail.com> Authored: Tue Sep 16 13:07:52 2014 -0500 Committer: Tyler Hobbs <ty...@datastax.com> Committed: Tue Sep 16 13:07:52 2014 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/cql3/QueryProcessor.java | 47 ++++++++++++++++- .../cassandra/metrics/CqlStatementMetrics.java | 54 ++++++++++++++++++++ .../apache/cassandra/service/ClientState.java | 2 +- 4 files changed, 101 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/19c6cc19/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3ee938a..cf7112c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.0.11: + * Add metrics for prepared statement usage and eviction (CASSANDRA-7930) * Make CQLSSTableWriter sync within partitions (CASSANDRA-7360) * Potentially use non-local replicas in CqlConfigHelper (CASSANDRA-7906) * Explicitly disallowing mixing multi-column and single-column http://git-wip-us.apache.org/repos/asf/cassandra/blob/19c6cc19/src/java/org/apache/cassandra/cql3/QueryProcessor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index a59fe9b..ee188a3 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -19,11 +19,14 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import com.google.common.primitives.Ints; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import com.googlecode.concurrentlinkedhashmap.EntryWeigher; +import com.googlecode.concurrentlinkedhashmap.EvictionListener; import org.antlr.runtime.*; import org.github.jamm.MemoryMeter; import org.slf4j.Logger; @@ -32,8 +35,10 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.cql3.statements.*; import org.apache.cassandra.db.*; import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.metrics.CqlStatementMetrics; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.thrift.ThriftClientState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; @@ -73,6 +78,9 @@ public class QueryProcessor implements QueryHandler private static final ConcurrentLinkedHashMap<MD5Digest, CQLStatement> preparedStatements; private static final ConcurrentLinkedHashMap<Integer, CQLStatement> thriftPreparedStatements; + public static final CqlStatementMetrics metrics = new CqlStatementMetrics(); + private static AtomicLong evictionCount = new AtomicLong(0); + static { if (MemoryMeter.isInitialized()) @@ -80,11 +88,29 @@ public class QueryProcessor implements QueryHandler preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>() .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY) .weigher(cqlMemoryUsageWeigher) - .build(); + .listener(new EvictionListener<MD5Digest, CQLStatement>() + { + @Override + public void onEviction(MD5Digest md5Digest, CQLStatement prepared) + { + metrics.activePreparedStatements.dec(); + metrics.evictedPreparedStatements.inc(); + evictionCount.incrementAndGet(); + } + }).build(); thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>() .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY) .weigher(thriftMemoryUsageWeigher) - .build(); + .listener(new EvictionListener<Integer, CQLStatement>() + { + @Override + public void onEviction(Integer i, CQLStatement prepared) + { + metrics.activePreparedStatements.dec(); + metrics.evictedPreparedStatements.inc(); + evictionCount.incrementAndGet(); + } + }).build(); } else { @@ -97,6 +123,17 @@ public class QueryProcessor implements QueryHandler .maximumWeightedCapacity(MAX_CACHE_PREPARED_COUNT) .build(); } + + StorageService.scheduledTasks.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + long count = evictionCount.getAndSet(0); + if (count > 0) + { + logger.info("{} prepared statements discarded in the last minute because cache limit reached (cache limit = {} bytes)", count, MAX_CACHE_PREPARED_MEMORY); + } + } + }, 1, 1, TimeUnit.MINUTES); } private QueryProcessor() @@ -172,6 +209,9 @@ public class QueryProcessor implements QueryHandler if (prepared.getBoundTerms() != options.getValues().size()) throw new InvalidRequestException("Invalid amount of bind variables"); + if (!queryState.getClientState().isInternal) + metrics.executedUnprepared.inc(); + return processStatement(prepared, queryState, options); } @@ -271,6 +311,7 @@ public class QueryProcessor implements QueryHandler { int statementId = toHash.hashCode(); thriftPreparedStatements.put(statementId, prepared.statement); + metrics.activePreparedStatements.inc(); logger.trace(String.format("Stored prepared statement #%d with %d bind markers", statementId, prepared.statement.getBoundTerms())); @@ -280,6 +321,7 @@ public class QueryProcessor implements QueryHandler { MD5Digest statementId = MD5Digest.compute(toHash); preparedStatements.put(statementId, prepared.statement); + metrics.activePreparedStatements.inc(); logger.trace(String.format("Stored prepared statement %s with %d bind markers", statementId, prepared.statement.getBoundTerms())); @@ -306,6 +348,7 @@ public class QueryProcessor implements QueryHandler logger.trace("[{}] '{}'", i+1, variables.get(i)); } + metrics.executedPrepared.inc(); return processStatement(statement, queryState, options); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/19c6cc19/src/java/org/apache/cassandra/metrics/CqlStatementMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/CqlStatementMetrics.java b/src/java/org/apache/cassandra/metrics/CqlStatementMetrics.java new file mode 100644 index 0000000..ba27d89 --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/CqlStatementMetrics.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.metrics; + +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.util.RatioGauge; + + +public class CqlStatementMetrics +{ + private final MetricNameFactory factory = new DefaultNameFactory("CqlStatement"); + public final Counter activePreparedStatements = Metrics.newCounter(factory.createMetricName("ActivePreparedStatements")); + public final Counter evictedPreparedStatements = Metrics.newCounter(factory.createMetricName("EvictedPreparedStatements")); + public final Counter executedPrepared = Metrics.newCounter(factory.createMetricName("ExecutedPrepared")); + public final Counter executedUnprepared = Metrics.newCounter(factory.createMetricName("ExecutedUnPrepared")); + + public final Gauge<Double> preparedRatio = Metrics.newGauge(factory.createMetricName("PreparedUnpreparedRatio"), new RatioGauge() + { + protected double getNumerator() + { + long num = executedPrepared.count(); + return num == 0 ? 1 : num; + } + + protected double getDenominator() + { + long den = executedUnprepared.count(); + return den == 0 ? 1 : den; + } + }); + + public void reset() + { + executedPrepared.clear(); + executedUnprepared.clear(); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/19c6cc19/src/java/org/apache/cassandra/service/ClientState.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java index be3b895..c14540d 100644 --- a/src/java/org/apache/cassandra/service/ClientState.java +++ b/src/java/org/apache/cassandra/service/ClientState.java @@ -102,7 +102,7 @@ public class ClientState // isInternal is used to mark ClientState as used by some internal component // that should have an ability to modify system keyspace. - private final boolean isInternal; + public final boolean isInternal; // The remote address of the client - null for internal clients. private final SocketAddress remoteAddress;
