Updated Branches: refs/heads/trunk 4c0be33ea -> deac4cc17
More detailed read repair metrics. Patch by Jingsi Zhu, reviewed by brandonwilliams for CASSANDRA-5618 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e301c38d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e301c38d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e301c38d Branch: refs/heads/trunk Commit: e301c38d3a592ff943ea0703403542dd2b7f499d Parents: b853630 Author: Brandon Williams <brandonwilli...@apache.org> Authored: Wed Jun 5 15:11:56 2013 -0500 Committer: Brandon Williams <brandonwilli...@apache.org> Committed: Wed Jun 5 15:11:56 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/metrics/ReadRepairMetrics.java | 25 +++++++++++++++ .../org/apache/cassandra/service/ReadCallback.java | 5 ++- .../org/apache/cassandra/service/StorageProxy.java | 25 ++++++++++++++- .../cassandra/service/StorageProxyMBean.java | 6 +++ src/java/org/apache/cassandra/tools/NodeCmd.java | 2 + src/java/org/apache/cassandra/tools/NodeProbe.java | 15 +++++++++ 7 files changed, 77 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index edbb94d..75b3173 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,7 @@ * cqlsh: fix COPY FROM with ReversedType (CASSANDRA-5610) * Allow creating CUSTOM indexes on collections (CASSANDRA-5615) * Evaluate now() function at execution time (CASSANDRA-5616) + * Expose detailed read repair metrics (CASSANDRA-5618) Merged from 1.1: * Remove buggy thrift max message length option (CASSANDRA-5529) * Fix NPE in Pig's widerow mode (CASSANDRA-5488) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java new file mode 100644 index 0000000..3f48fee --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java @@ -0,0 +1,25 @@ +package org.apache.cassandra.metrics; + +import java.util.concurrent.TimeUnit; + +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.Meter; +import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.util.RatioGauge; + +/** + * Metrics related to Read Repair. + */ +public class ReadRepairMetrics { + public static final String GROUP_NAME = "org.apache.cassandra.metrics"; + public static final String TYPE_NAME = "ReadRepair"; + + public static final Meter repairedBlocking = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "RepairedBlocking"), "RepairedBlocking", TimeUnit.SECONDS); + public static final Meter repairedBackground = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "RepairedBackground"), "RepairedBackground", TimeUnit.SECONDS); + public static final Meter attempted = + Metrics.newMeter(new MetricName(GROUP_NAME, TYPE_NAME, "Attempted"), "Attempted", TimeUnit.SECONDS); +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/service/ReadCallback.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/ReadCallback.java b/src/java/org/apache/cassandra/service/ReadCallback.java index a19df5f..92032f2 100644 --- a/src/java/org/apache/cassandra/service/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/ReadCallback.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.Table; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.IAsyncCallback; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; @@ -174,7 +175,9 @@ public class ReadCallback<TMessage, TResolved> implements IAsyncCallback<TMessag if (logger.isDebugEnabled()) logger.debug("Digest mismatch:", e); - + + ReadRepairMetrics.repairedBackground.mark(); + ReadCommand readCommand = (ReadCommand) command; final RowDataResolver repairResolver = new RowDataResolver(readCommand.table, readCommand.key, readCommand.filter()); IAsyncCallback repairHandler = new AsyncRepairCallback(repairResolver, endpoints.size()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/service/StorageProxy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index e8440c4..5517387 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -39,6 +39,7 @@ import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.ReadRepairDecision; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; import org.apache.cassandra.db.Table; @@ -59,6 +60,7 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.metrics.ClientRequestMetrics; +import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.*; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; @@ -888,7 +890,13 @@ public class StorageProxy implements StorageProxyMBean List<InetAddress> endpoints = getLiveSortedEndpoints(table, command.key); CFMetaData cfm = Schema.instance.getCFMetaData(command.getKeyspace(), command.getColumnFamilyName()); - endpoints = consistency_level.filterForQuery(table, endpoints, cfm.newReadRepairDecision()); + + ReadRepairDecision rrDecision = cfm.newReadRepairDecision(); + endpoints = consistency_level.filterForQuery(table, endpoints, rrDecision); + + if (rrDecision != ReadRepairDecision.NONE) { + ReadRepairMetrics.attempted.mark(); + } RowDigestResolver resolver = new RowDigestResolver(command.table, command.key); ReadCallback<ReadResponse, Row> handler = new ReadCallback(resolver, consistency_level, command, endpoints); @@ -960,6 +968,9 @@ public class StorageProxy implements StorageProxyMBean catch (DigestMismatchException ex) { logger.debug("Digest mismatch: {}", ex.toString()); + + ReadRepairMetrics.repairedBlocking.mark(); + // Do a full data read to resolve the correct response (and repair node that need be) RowDataResolver resolver = new RowDataResolver(command.table, command.key, command.filter()); ReadCallback<ReadResponse, Row> repairHandler = handler.withNewResolver(resolver); @@ -1702,4 +1713,16 @@ public class StorageProxy implements StorageProxyMBean public Long getTruncateRpcTimeout() { return DatabaseDescriptor.getTruncateRpcTimeout(); } public void setTruncateRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setTruncateRpcTimeout(timeoutInMillis); } + + public long getReadRepairAttempted() { + return ReadRepairMetrics.attempted.count(); + } + + public long getReadRepairRepairedBlocking() { + return ReadRepairMetrics.repairedBlocking.count(); + } + + public long getReadRepairRepairedBackground() { + return ReadRepairMetrics.repairedBackground.count(); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/service/StorageProxyMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java index 1bbfd1f..e8f5b4a 100644 --- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java +++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.service; +import org.apache.cassandra.metrics.ReadRepairMetrics; + public interface StorageProxyMBean { /** @@ -86,4 +88,8 @@ public interface StorageProxyMBean public void setRangeRpcTimeout(Long timeoutInMillis); public Long getTruncateRpcTimeout(); public void setTruncateRpcTimeout(Long timeoutInMillis); + + public long getReadRepairAttempted(); + public long getReadRepairRepairedBlocking(); + public long getReadRepairRepairedBackground(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/tools/NodeCmd.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index bca0fcd..1cb30ad 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -669,6 +669,8 @@ public class NodeCmd outs.printf(" Error retrieving file data for %s%n", host); } } + + outs.printf("Read Repair Statistics:%nAttempted: %d%nMismatch (Blocking): %d%nMismatch (Background): %d%n", probe.getReadRepairAttempted(), probe.getReadRepairRepairedBlocking(), probe.getReadRepairRepairedBackground()); MessagingServiceMBean ms = probe.msProxy; outs.printf("%-25s", "Pool Name"); http://git-wip-us.apache.org/repos/asf/cassandra/blob/e301c38d/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 5db8f1c..dbcb66e 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -825,6 +825,21 @@ public class NodeProbe { return failed; } + + public long getReadRepairAttempted() + { + return spProxy.getReadRepairAttempted(); + } + + public long getReadRepairRepairedBlocking() + { + return spProxy.getReadRepairRepairedBlocking(); + } + + public long getReadRepairRepairedBackground() + { + return spProxy.getReadRepairRepairedBackground(); + } } class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>