Log compaction merge counts; patch by yukim reviewed by jbellis for CASSANDRA-4894
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0207bdf5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0207bdf5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0207bdf5 Branch: refs/heads/cassandra-1.2 Commit: 0207bdf5ff07b60d9a44f7d592061a7b74d034e1 Parents: fca00ac Author: Yuki Morishita <yu...@apache.org> Authored: Wed Dec 19 08:56:52 2012 -0600 Committer: Yuki Morishita <yu...@apache.org> Committed: Wed Dec 19 08:56:52 2012 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../db/compaction/AbstractCompactionIterable.java | 24 +++++++++++++++ .../db/compaction/CompactionIterable.java | 1 + .../cassandra/db/compaction/CompactionTask.java | 4 +- .../db/compaction/ParallelCompactionIterable.java | 1 + 5 files changed, 29 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0207bdf5/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d3a0077..5f1ff2b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ * Add debug logging to list filenames processed by Directories.migrateFile method (CASSANDRA-4939) * Expose black-listed directories via JMX (CASSANDRA-4848) + * Log compaction merge counts (CASSANDRA-4894) 1.2.0-rc2 http://git-wip-us.apache.org/repos/asf/cassandra/blob/0207bdf5/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java index def7191..d7b9f63 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db.compaction; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.cassandra.utils.CloseableIterator; @@ -28,6 +29,12 @@ public abstract class AbstractCompactionIterable extends CompactionInfo.Holder i protected final long totalBytes; protected volatile long bytesRead; protected final List<ICompactionScanner> scanners; + /* + * counters for merged rows. + * array index represents (number of merged rows - 1), so index 0 is counter for no merge (1 row), + * index 1 is counter for 2 rows merged, and so on. + */ + protected final AtomicInteger[] mergeCounters; public AbstractCompactionIterable(CompactionController controller, OperationType type, List<ICompactionScanner> scanners) { @@ -40,6 +47,9 @@ public abstract class AbstractCompactionIterable extends CompactionInfo.Holder i for (ICompactionScanner scanner : scanners) bytes += scanner.getLengthInBytes(); this.totalBytes = bytes; + mergeCounters = new AtomicInteger[scanners.size()]; + for (int i = 0; i < mergeCounters.length; i++) + mergeCounters[i] = new AtomicInteger(); } public CompactionInfo getCompactionInfo() @@ -50,5 +60,19 @@ public abstract class AbstractCompactionIterable extends CompactionInfo.Holder i totalBytes); } + protected void updateCounterFor(int rows) + { + assert rows > 0 && rows - 1 < mergeCounters.length; + mergeCounters[rows - 1].incrementAndGet(); + } + + public int[] getMergedRowCounts() + { + int[] counters = new int[mergeCounters.length]; + for (int i = 0; i < counters.length; i++) + counters[i] = mergeCounters[i].get(); + return counters; + } + public abstract CloseableIterator<AbstractCompactedRow> iterator(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0207bdf5/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java index f095a57..9f4f7a0 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java @@ -67,6 +67,7 @@ public class CompactionIterable extends AbstractCompactionIterable { assert !rows.isEmpty(); + CompactionIterable.this.updateCounterFor(rows.size()); try { AbstractCompactedRow compactedRow = controller.getCompactedRow(new ArrayList<SSTableIdentityIterator>(rows)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/0207bdf5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 4eeeaf2..481b7f4 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -238,8 +238,8 @@ public class CompactionTask extends AbstractCompactionTask builder.append("]"); double mbps = dTime > 0 ? (double)endsize/(1024*1024)/((double)dTime/1000) : 0; - logger.info(String.format("Compacted to %s. %,d to %,d (~%d%% of original) bytes for %,d keys at %fMB/s. Time: %,dms.", - builder.toString(), startsize, endsize, (int) (ratio * 100), totalkeysWritten, mbps, dTime)); + logger.info(String.format("Compacted to %s. %,d to %,d (~%d%% of original) bytes for %,d keys at %fMB/s. Time: %,dms. Compaction merge counts: %s.", + builder.toString(), startsize, endsize, (int) (ratio * 100), totalkeysWritten, mbps, dTime, Arrays.toString(ci.getMergedRowCounts()))); logger.debug(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize))); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0207bdf5/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java index 8f04205..eaf35f2 100644 --- a/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java @@ -158,6 +158,7 @@ public class ParallelCompactionIterable extends AbstractCompactionIterable { assert rows.size() > 0; + ParallelCompactionIterable.this.updateCounterFor(rows.size()); CompactedRowContainer compacted = getCompactedRow(rows); rows.clear(); if ((row++ % 1000) == 0)