Repository: cassandra Updated Branches: refs/heads/cassandra-2.1 fd6f9c6f9 -> 97da271b2
Re-enable CASSANDRA-7688 - properly handle empty intervalTree in DT.View#sstablesInBounds - switch gc gs to 0 for system.size_estimates table - add a -D param to alter the estimates update interval - log the duration of the estimation process Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/97da271b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/97da271b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/97da271b Branch: refs/heads/cassandra-2.1 Commit: 97da271b29941596da7b401c2ed7352de111dc10 Parents: fd6f9c6 Author: Aleksey Yeschenko <alek...@apache.org> Authored: Tue Feb 10 03:54:22 2015 +0300 Committer: Aleksey Yeschenko <alek...@apache.org> Committed: Tue Feb 10 03:54:22 2015 +0300 ---------------------------------------------------------------------- CHANGES.txt | 4 ++++ .../org/apache/cassandra/config/CFMetaData.java | 3 ++- .../org/apache/cassandra/db/DataTracker.java | 2 ++ .../cassandra/db/SizeEstimatesRecorder.java | 20 +++++++++++++++----- .../cassandra/service/CassandraDaemon.java | 5 ++++- 5 files changed, 27 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/97da271b/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d44d70d..c5cff48 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +2.1.4 + * Write partition size estimates into a system table (CASSANDRA-7688) + + 2.1.3 * Upgrade libthrift to 0.9.2 (CASSANDRA-8685) * Don't use the shared ref in sstableloader (CASSANDRA-8704) http://git-wip-us.apache.org/repos/asf/cassandra/blob/97da271b/src/java/org/apache/cassandra/config/CFMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index d55d1c0..8bb5ccf 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -298,7 +298,8 @@ public final class CFMetaData + "mean_partition_size bigint," + "partitions_count bigint," + "PRIMARY KEY ((keyspace_name), table_name, range_start, range_end)" - + ") WITH COMMENT='per-table primary range size estimates'"); + + ") WITH COMMENT='per-table primary range size estimates' " + + "AND gc_grace_seconds=0"); public static class SpeculativeRetry http://git-wip-us.apache.org/repos/asf/cassandra/blob/97da271b/src/java/org/apache/cassandra/db/DataTracker.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java index f672cf2..5ec06bc 100644 --- a/src/java/org/apache/cassandra/db/DataTracker.java +++ b/src/java/org/apache/cassandra/db/DataTracker.java @@ -758,6 +758,8 @@ public class DataTracker public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds) { + if (intervalTree.isEmpty()) + return Collections.emptyList(); RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right; return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree)); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/97da271b/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java b/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java index b7e5715..1472c11 100644 --- a/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java +++ b/src/java/org/apache/cassandra/db/SizeEstimatesRecorder.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db; import java.util.*; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,8 +62,18 @@ public class SizeEstimatesRecorder extends MigrationListener implements Runnable Collection<Range<Token>> localRanges = StorageService.instance.getTokenMetadata().getPrimaryRangesFor(localTokens); for (Keyspace keyspace : Keyspace.nonSystem()) + { for (ColumnFamilyStore table : keyspace.getColumnFamilyStores()) + { + long start = System.nanoTime(); recordSizeEstimates(table, localRanges); + long passed = System.nanoTime() - start; + logger.debug("Spent {} milliseconds on estimating {}.{} size", + TimeUnit.NANOSECONDS.toMillis(passed), + table.metadata.ksName, + table.metadata.cfName); + } + } } private void recordSizeEstimates(ColumnFamilyStore table, Collection<Range<Token>> localRanges) @@ -72,20 +83,19 @@ public class SizeEstimatesRecorder extends MigrationListener implements Runnable for (Range<Token> range : localRanges) { // filter sstables that have partitions in this range. - List<SSTableReader> sstables = null; Refs<SSTableReader> refs = null; while (refs == null) { - sstables = table.viewFilter(range.toRowBounds()).apply(table.getDataTracker().getView()); - refs = Refs.tryRef(sstables); + ColumnFamilyStore.ViewFragment view = table.select(table.viewFilter(range.toRowBounds())); + refs = Refs.tryRef(view.sstables); } long partitionsCount, meanPartitionSize; try { // calculate the estimates. - partitionsCount = estimatePartitionsCount(sstables, range); - meanPartitionSize = estimateMeanPartitionSize(sstables); + partitionsCount = estimatePartitionsCount(refs, range); + meanPartitionSize = estimateMeanPartitionSize(refs); } finally { http://git-wip-us.apache.org/repos/asf/cassandra/blob/97da271b/src/java/org/apache/cassandra/service/CassandraDaemon.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 50c8295..67d737e 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -366,7 +366,10 @@ public class CassandraDaemon waitForGossipToSettle(); // schedule periodic dumps of table size estimates into SystemKeyspace.SIZE_ESTIMATES_CF - // ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, 5 * 60, TimeUnit.SECONDS); + // set cassandra.size_recorder_interval to 0 to disable + int sizeRecorderInterval = Integer.getInteger("cassandra.size_recorder_interval", 5 * 60); + if (sizeRecorderInterval > 0) + ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS); // Thrift InetAddress rpcAddr = DatabaseDescriptor.getRpcAddress();