Repository: cassandra Updated Branches: refs/heads/trunk 3b6edc6af -> 41a35ec74
Do more aggressive ttl expiration checks to be able to drop more sstables Patch by Bjorn Hegerfors; reviewed by marcuse and slebresne for CASSANDRA-8243 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/abbcfc5f Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/abbcfc5f Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/abbcfc5f Branch: refs/heads/trunk Commit: abbcfc5f72323d0c6040a3cc02aba8f2c0058d95 Parents: 054beee Author: Marcus Eriksson <marc...@apache.org> Authored: Fri Nov 14 10:54:39 2014 +0100 Committer: Marcus Eriksson <marc...@apache.org> Committed: Fri Nov 14 11:02:45 2014 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../db/compaction/CompactionController.java | 20 ++++---- .../cassandra/db/compaction/TTLExpiryTest.java | 50 ++++++++++++++++++++ 3 files changed, 60 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6228893..2476d25 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.3 + * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243) * Add more log info if readMeter is null (CASSANDRA-8238) * add check of the system wall clock time at startup (CASSANDRA-8305) * Support for frozen collections (CASSANDRA-7859) http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/src/java/org/apache/cassandra/db/compaction/CompactionController.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java index ef27805..f23d39a 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java @@ -92,12 +92,11 @@ public class CompactionController implements AutoCloseable * Finds expired sstables * * works something like this; - * 1. find "global" minTimestamp of overlapping sstables (excluding the possibly droppable ones) - * 2. build a list of candidates to be dropped - * 3. sort the candidate list, biggest maxTimestamp first in list - * 4. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp) and it is included in the compaction - * - if not droppable, update global minTimestamp and remove from candidates - * 5. return candidates. + * 1. find "global" minTimestamp of overlapping sstables and compacting sstables containing any non-expired data + * 2. build a list of fully expired candidates + * 3. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp) + * - if not droppable, remove from candidates + * 4. return candidates. * * @param cfStore * @param compacting we take the drop-candidates from this set, it is usually the sstables included in the compaction @@ -127,10 +126,10 @@ public class CompactionController implements AutoCloseable minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp()); } - // we still need to keep candidates that might shadow something in a - // non-candidate sstable. And if we remove a sstable from the candidates, we - // must take it's timestamp into account (hence the sorting below). - Collections.sort(candidates, SSTableReader.maxTimestampComparator); + // At this point, minTimestamp denotes the lowest timestamp of any relevant + // SSTable that contains a constructive value. candidates contains all the + // candidates with no constructive values. The ones out of these that have + // (getMaxTimestamp() < minTimestamp) serve no purpose anymore. Iterator<SSTableReader> iterator = candidates.iterator(); while (iterator.hasNext()) @@ -138,7 +137,6 @@ public class CompactionController implements AutoCloseable SSTableReader candidate = iterator.next(); if (candidate.getMaxTimestamp() >= minTimestamp) { - minTimestamp = Math.min(candidate.getMinTimestamp(), minTimestamp); iterator.remove(); } else http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java index b98af68..4fe5cfb 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction; * */ +import com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,6 +32,10 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.io.sstable.SSTableReader; import org.apache.cassandra.io.sstable.SSTableScanner; import org.apache.cassandra.utils.ByteBufferUtil; + +import java.util.Collections; +import java.util.Set; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -38,6 +43,51 @@ import static org.junit.Assert.assertTrue; public class TTLExpiryTest extends SchemaLoader { @Test + public void testAggressiveFullyExpired() + { + ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); + cfs.disableAutoCompaction(); + cfs.metadata.gcGraceSeconds(0); + + DecoratedKey ttlKey = Util.dk("ttl"); + Mutation rm = new Mutation("Keyspace1", ttlKey.getKey()); + rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1); + rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + + rm = new Mutation("Keyspace1", ttlKey.getKey()); + rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1); + rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + + rm = new Mutation("Keyspace1", ttlKey.getKey()); + rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1); + rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + + rm = new Mutation("Keyspace1", ttlKey.getKey()); + rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3); + rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + + Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables()); + int now = (int)(System.currentTimeMillis() / 1000); + int gcBefore = now + 2; + Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables( + cfs, + sstables, + Collections.EMPTY_SET, + gcBefore); + assertEquals(2, expired.size()); + + cfs.clearUnsafe(); + } + + @Test public void testSimpleExpire() throws InterruptedException { ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
