Repository: cassandra Updated Branches: refs/heads/trunk 30a704e01 -> 91e3d3421
Add option to change how often we check for expired sstables with DTCS Patch by marcuse; reviewed by yukim for CASSANDRA-9882 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19 Branch: refs/heads/trunk Commit: 17042f19b084d93bd3bcc35c7282812f456b7640 Parents: f53aacb Author: Marcus Eriksson <marc...@apache.org> Authored: Mon Aug 17 09:50:37 2015 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Tue Aug 18 10:47:04 2015 +0200 ---------------------------------------------------------------------- .../DateTieredCompactionStrategy.java | 7 +++---- .../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++ .../DateTieredCompactionStrategyTest.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java index fea4995..2e43352 100644 --- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java @@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy protected DateTieredCompactionStrategyOptions options; protected volatile int estimatedRemainingTasks; - @VisibleForTesting - long lastExpiredCheck; + private long lastExpiredCheck; public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options) { @@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables(); Set<SSTableReader> expired = Collections.emptySet(); - // we only check for expired sstables every 10 minutes due to it being an expensive operation - if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10)) + // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation + if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency) { // Find fully expired SSTables. Those will be included no matter what. expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore); http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java index f54c020..0cbf90e 100644 --- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java +++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java @@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION = TimeUnit.MICROSECONDS; protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365; protected static final long DEFAULT_BASE_TIME_SECONDS = 60; + protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10; protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution"; protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days"; protected static final String BASE_TIME_KEY = "base_time_seconds"; + protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds"; protected final long maxSSTableAge; protected final long baseTime; + protected final long expiredSSTableCheckFrequency; public DateTieredCompactionStrategyOptions(Map<String, String> options) { @@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS)); optionValue = options.get(BASE_TIME_KEY); baseTime = timestampResolution.convert(optionValue == null ? DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS); + optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY); + expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS); } public DateTieredCompactionStrategyOptions() { maxSSTableAge = Math.round(DEFAULT_MAX_SSTABLE_AGE_DAYS * DEFAULT_TIMESTAMP_RESOLUTION.convert(1, TimeUnit.DAYS)); baseTime = DEFAULT_TIMESTAMP_RESOLUTION.convert(DEFAULT_BASE_TIME_SECONDS, TimeUnit.SECONDS); + expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS); } public static Map<String, String> validateOptions(Map<String, String> options, Map<String, String> uncheckedOptions) throws ConfigurationException @@ -92,9 +98,24 @@ public final class DateTieredCompactionStrategyOptions throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, BASE_TIME_KEY), e); } + optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY); + try + { + long expiredCheckFrequency = optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue); + if (expiredCheckFrequency < 0) + { + throw new ConfigurationException(String.format("%s must not be negative, but was %d", EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, expiredCheckFrequency)); + } + } + catch (NumberFormatException e) + { + throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY), e); + } + uncheckedOptions.remove(MAX_SSTABLE_AGE_KEY); uncheckedOptions.remove(BASE_TIME_KEY); uncheckedOptions.remove(TIMESTAMP_RESOLUTION_KEY); + uncheckedOptions.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY); return uncheckedOptions; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java index 0084a16..6d9bfc2 100644 --- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java @@ -305,10 +305,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30"); options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS"); options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60)))); + options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0"); DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options); dtcs.startup(); assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000))); - dtcs.lastExpiredCheck = 0; Thread.sleep(2000); AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000)); assertNotNull(t);
