Updated Branches: refs/heads/trunk d11ba8ab8 -> 6968f68cd
allow STCS options to apply to the L0 compaction performed by LCS patch by Carl Yeksigian; reviewed by jbellis for CASSANDRA-5439 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6968f68c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6968f68c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6968f68c Branch: refs/heads/trunk Commit: 6968f68cd7c8d4a1ab4a6d4638c547e9de33a560 Parents: d11ba8a Author: Jonathan Ellis <jbel...@apache.org> Authored: Sun Apr 21 18:08:12 2013 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Sun Apr 21 18:08:12 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 3 +- .../db/compaction/LeveledCompactionStrategy.java | 5 +- .../cassandra/db/compaction/LeveledManifest.java | 18 ++-- .../compaction/SizeTieredCompactionStrategy.java | 65 +------------- .../apache/cassandra/tools/StandaloneScrubber.java | 1 + 5 files changed, 22 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/6968f68c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2f5f5a9..9551c4c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,8 @@ 2.0 * removed PBSPredictor (CASSANDRA-5455) * CAS support (CASSANDRA-5062, ) - * Leveled compaction performs size-tiered compactions in L0 (CASSANDRA-5371) + * Leveled compaction performs size-tiered compactions in L0 + (CASSANDRA-5371, 5439) * Add yaml network topology snitch for mixed ec2/other envs (CASSANDRA-5339) * Log when a node is down longer than the hint window (CASSANDRA-4554) * Optimize tombstone creation for ExpiringColumns (CASSANDRA-4917) http://git-wip-us.apache.org/repos/asf/cassandra/blob/6968f68c/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java index f704518..0e8c2a7 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java @@ -53,6 +53,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem { super(cfs, options); int configuredMaxSSTableSize = 5; + SizeTieredCompactionStrategyOptions localOptions = new SizeTieredCompactionStrategyOptions(options); if (options != null) { String value = options.containsKey(SSTABLE_SIZE_OPTION) ? options.get(SSTABLE_SIZE_OPTION) : "5"; @@ -63,7 +64,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem cfs.getDataTracker().subscribe(this); logger.debug("{} subscribed to the data tracker.", this); - manifest = LeveledManifest.create(cfs, this.maxSSTableSizeInMB); + manifest = LeveledManifest.create(cfs, this.maxSSTableSizeInMB, Collections.<SSTableReader>emptyList(), localOptions); logger.debug("Created {}", manifest); } @@ -347,6 +348,8 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem uncheckedOptions.remove(SSTABLE_SIZE_OPTION); + uncheckedOptions = SizeTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions); + return uncheckedOptions; } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/6968f68c/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 8ae0834..fb4244d 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -57,11 +57,13 @@ public class LeveledManifest private final List<SSTableReader>[] generations; private final RowPosition[] lastCompactedKeys; private final int maxSSTableSizeInBytes; + private final SizeTieredCompactionStrategyOptions options; - private LeveledManifest(ColumnFamilyStore cfs, int maxSSTableSizeInMB) + private LeveledManifest(ColumnFamilyStore cfs, int maxSSTableSizeInMB, SizeTieredCompactionStrategyOptions options) { this.cfs = cfs; this.maxSSTableSizeInBytes = maxSSTableSizeInMB * 1024 * 1024; + this.options = options; // allocate enough generations for a PB of data int n = (int) Math.log10(1000 * 1000 * 1000 / maxSSTableSizeInMB); @@ -74,14 +76,14 @@ public class LeveledManifest } } - static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize) + public static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize, List<SSTableReader> sstables) { - return create(cfs, maxSSTableSize, cfs.getSSTables()); + return create(cfs, maxSSTableSize, sstables, new SizeTieredCompactionStrategyOptions()); } - public static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize, Iterable<SSTableReader> sstables) + public static LeveledManifest create(ColumnFamilyStore cfs, int maxSSTableSize, Iterable<SSTableReader> sstables, SizeTieredCompactionStrategyOptions options) { - LeveledManifest manifest = new LeveledManifest(cfs, maxSSTableSize); + LeveledManifest manifest = new LeveledManifest(cfs, maxSSTableSize, options); // ensure all SSTables are in the manifest for (SSTableReader ssTableReader : sstables) @@ -271,9 +273,9 @@ public class LeveledManifest Iterable<SSTableReader> candidates = cfs.getDataTracker().getUncompactingSSTables(generations[0]); List<Pair<SSTableReader,Long>> pairs = SizeTieredCompactionStrategy.createSSTableAndLengthPairs(AbstractCompactionStrategy.filterSuspectSSTables(candidates)); List<List<SSTableReader>> buckets = SizeTieredCompactionStrategy.getBuckets(pairs, - SizeTieredCompactionStrategy.DEFAULT_BUCKET_HIGH, - SizeTieredCompactionStrategy.DEFAULT_BUCKET_LOW, - SizeTieredCompactionStrategy.DEFAULT_MIN_SSTABLE_SIZE); + options.bucketHigh, + options.bucketLow, + options.minSSTableSize); List<SSTableReader> mostInteresting = SizeTieredCompactionStrategy.mostInterestingBucket(buckets, 4, 32); if (!mostInteresting.isEmpty()) return Pair.create(mostInteresting, 0); http://git-wip-us.apache.org/repos/asf/cassandra/blob/6968f68c/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java index ae6f627..3678184 100644 --- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java @@ -34,28 +34,15 @@ import org.apache.cassandra.utils.Pair; public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy { private static final Logger logger = LoggerFactory.getLogger(SizeTieredCompactionStrategy.class); - protected static final long DEFAULT_MIN_SSTABLE_SIZE = 50L * 1024L * 1024L; - protected static final double DEFAULT_BUCKET_LOW = 0.5; - protected static final double DEFAULT_BUCKET_HIGH = 1.5; - protected static final String MIN_SSTABLE_SIZE_KEY = "min_sstable_size"; - protected static final String BUCKET_LOW_KEY = "bucket_low"; - protected static final String BUCKET_HIGH_KEY = "bucket_high"; - - protected long minSSTableSize; - protected double bucketLow; - protected double bucketHigh; + + protected SizeTieredCompactionStrategyOptions options; protected volatile int estimatedRemainingTasks; public SizeTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options) { super(cfs, options); this.estimatedRemainingTasks = 0; - String optionValue = options.get(MIN_SSTABLE_SIZE_KEY); - minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue); - optionValue = options.get(BUCKET_LOW_KEY); - bucketLow = optionValue == null ? DEFAULT_BUCKET_LOW : Double.parseDouble(optionValue); - optionValue = options.get(BUCKET_HIGH_KEY); - bucketHigh = optionValue == null ? DEFAULT_BUCKET_HIGH : Double.parseDouble(optionValue); + this.options = new SizeTieredCompactionStrategyOptions(options); } private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore) @@ -68,7 +55,7 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy int maxThreshold = cfs.getMaximumCompactionThreshold(); Set<SSTableReader> candidates = cfs.getUncompactingSSTables(); - List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(filterSuspectSSTables(candidates)), bucketHigh, bucketLow, minSSTableSize); + List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(filterSuspectSSTables(candidates)), options.bucketHigh, options.bucketLow, options.minSSTableSize); logger.debug("Compaction buckets are {}", buckets); updateEstimatedCompactionsByTasks(buckets); List<SSTableReader> mostInteresting = mostInterestingBucket(buckets, minThreshold, maxThreshold); @@ -252,50 +239,8 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException { Map<String, String> uncheckedOptions = AbstractCompactionStrategy.validateOptions(options); + uncheckedOptions = SizeTieredCompactionStrategyOptions.validateOptions(options, uncheckedOptions); - String optionValue = options.get(MIN_SSTABLE_SIZE_KEY); - try - { - long minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue); - if (minSSTableSize < 0) - { - throw new ConfigurationException(String.format("%s must be non negative: %d", MIN_SSTABLE_SIZE_KEY, minSSTableSize)); - } - } - catch (NumberFormatException e) - { - throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, MIN_SSTABLE_SIZE_KEY), e); - } - - double bucketLow, bucketHigh; - optionValue = options.get(BUCKET_LOW_KEY); - try - { - bucketLow = optionValue == null ? DEFAULT_BUCKET_LOW : Double.parseDouble(optionValue); - } - catch (NumberFormatException e) - { - throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, DEFAULT_BUCKET_LOW), e); - } - - optionValue = options.get(BUCKET_HIGH_KEY); - try - { - bucketHigh = optionValue == null ? DEFAULT_BUCKET_HIGH : Double.parseDouble(optionValue); - } - catch (NumberFormatException e) - { - throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, DEFAULT_BUCKET_HIGH), e); - } - - if (bucketHigh <= bucketLow) - { - throw new ConfigurationException(String.format("Bucket high value (%s) is less than or equal bucket low value (%s)", bucketHigh, bucketLow)); - } - - uncheckedOptions.remove(MIN_SSTABLE_SIZE_KEY); - uncheckedOptions.remove(BUCKET_LOW_KEY); - uncheckedOptions.remove(BUCKET_HIGH_KEY); uncheckedOptions.remove(CFPropDefs.KW_MINCOMPACTIONTHRESHOLD); uncheckedOptions.remove(CFPropDefs.KW_MAXCOMPACTIONTHRESHOLD); http://git-wip-us.apache.org/repos/asf/cassandra/blob/6968f68c/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 34a8a72..c9d39bb 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.util.*; +import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategyOptions; import org.apache.commons.cli.*; import org.apache.cassandra.config.DatabaseDescriptor;
