Merge branch 'cassandra-2.0' into cassandra-2.1.0 Conflicts: src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d1f38cd2 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d1f38cd2 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d1f38cd2 Branch: refs/heads/trunk Commit: d1f38cd27fc4c2ec7e74c70d174ca5a021f660bd Parents: 889f3ac 61543b4 Author: Marcus Eriksson <marc...@apache.org> Authored: Tue Aug 26 15:35:53 2014 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Tue Aug 26 15:36:42 2014 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../db/compaction/LazilyCompactedRow.java | 7 +-- .../apache/cassandra/db/RangeTombstoneTest.java | 56 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1f38cd2/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 80747dc,7a59744..a9ab462 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -183,46 -85,21 +183,47 @@@ Merged from 1.2 are thrown while handling native protocol messages (CASSANDRA-7470) * Fix row size miscalculation in LazilyCompactedRow (CASSANDRA-7543) * Fix race in background compaction check (CASSANDRA-7745) + * Don't clear out range tombstones during compaction (CASSANDRA-7808) -2.0.9 - * Fix CC#collectTimeOrderedData() tombstone optimisations (CASSANDRA-7394) - * Fix assertion error in CL.ANY timeout handling (CASSANDRA-7364) - * Handle empty CFs in Memtable#maybeUpdateLiveRatio() (CASSANDRA-7401) - * Fix native protocol CAS batches (CASSANDRA-7337) - * Add per-CF range read request latency metrics (CASSANDRA-7338) - * Fix NPE in StreamTransferTask.createMessageForRetry() (CASSANDRA-7323) - * Add conditional CREATE/DROP USER support (CASSANDRA-7264) - * Swap local and global default read repair chances (CASSANDRA-7320) - * Add missing iso8601 patterns for date strings (CASSANDRA-6973) - * Support selecting multiple rows in a partition using IN (CASSANDRA-6875) - * cqlsh: always emphasize the partition key in DESC output (CASSANDRA-7274) +2.1.0-rc1 + * Revert flush directory (CASSANDRA-6357) + * More efficient executor service for fast operations (CASSANDRA-4718) + * Move less common tools into a new cassandra-tools package (CASSANDRA-7160) + * Support more concurrent requests in native protocol (CASSANDRA-7231) + * Add tab-completion to debian nodetool packaging (CASSANDRA-6421) + * Change concurrent_compactors defaults (CASSANDRA-7139) + * Add PowerShell Windows launch scripts (CASSANDRA-7001) + * Make commitlog archive+restore more robust (CASSANDRA-6974) + * Fix marking commitlogsegments clean (CASSANDRA-6959) + * Add snapshot "manifest" describing files included (CASSANDRA-6326) + * Parallel streaming for sstableloader (CASSANDRA-3668) + * Fix bugs in supercolumns handling (CASSANDRA-7138) + * Fix ClassClassException on composite dense tables (CASSANDRA-7112) + * Cleanup and optimize collation and slice iterators (CASSANDRA-7107) + * Upgrade NBHM lib (CASSANDRA-7128) + * Optimize netty server (CASSANDRA-6861) + * Fix repair hang when given CF does not exist (CASSANDRA-7189) + * Allow c* to be shutdown in an embedded mode (CASSANDRA-5635) + * Add server side batching to native transport (CASSANDRA-5663) + * Make batchlog replay asynchronous (CASSANDRA-6134) + * remove unused classes (CASSANDRA-7197) + * Limit user types to the keyspace they are defined in (CASSANDRA-6643) + * Add validate method to CollectionType (CASSANDRA-7208) + * New serialization format for UDT values (CASSANDRA-7209, CASSANDRA-7261) + * Fix nodetool netstats (CASSANDRA-7270) + * Fix potential ClassCastException in HintedHandoffManager (CASSANDRA-7284) + * Use prepared statements internally (CASSANDRA-6975) + * Fix broken paging state with prepared statement (CASSANDRA-7120) + * Fix IllegalArgumentException in CqlStorage (CASSANDRA-7287) + * Allow nulls/non-existant fields in UDT (CASSANDRA-7206) + * Backport Thrift MultiSliceRequest (CASSANDRA-7027) + * Handle overlapping MultiSlices (CASSANDRA-7279) + * Fix DataOutputTest on Windows (CASSANDRA-7265) + * Embedded sets in user defined data-types are not updating (CASSANDRA-7267) + * Add tuple type to CQL/native protocol (CASSANDRA-7248) + * Fix CqlPagingRecordReader on tables with few rows (CASSANDRA-7322) +Merged from 2.0: * Copy compaction options to make sure they are reloaded (CASSANDRA-7290) * Add option to do more aggressive tombstone compactions (CASSANDRA-6563) * Don't try to compact already-compacting files in HHOM (CASSANDRA-7288) http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1f38cd2/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java index 7d6f872,e3f18bd..1f8c7dc --- a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java +++ b/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java @@@ -254,26 -263,20 +254,27 @@@ public class LazilyCompactedRow extend } else { + boolean shouldPurge = container.getSortedColumns().iterator().next().timestamp() < maxPurgeableTimestamp; // when we clear() the container, it removes the deletion info, so this needs to be reset each time container.delete(maxRowTombstone); - ColumnFamily purged = PrecompactedRow.removeDeleted(key, shouldPurge, controller, container); - if (purged == null || !purged.iterator().hasNext()) + removeDeleted(container, shouldPurge, key, controller); + Iterator<Cell> iter = container.iterator(); + if (!iter.hasNext()) { - container.clear(); + // don't call clear() because that resets the deletion time. See CASSANDRA-7808. - container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata());; ++ container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata()); return null; } - Column reduced = purged.iterator().next(); + + int localDeletionTime = container.deletionInfo().getTopLevelDeletion().localDeletionTime; + if (localDeletionTime < Integer.MAX_VALUE) + tombstones.update(localDeletionTime); + + Cell reduced = iter.next(); - container.clear(); + container = ArrayBackedSortedColumns.factory.create(emptyColumnFamily.metadata()); - // PrecompactedRow.removeDeleted has only checked the top-level CF deletion times, - // not the range tombstones. For that we use the columnIndexer tombstone tracker. + // removeDeleted have only checked the top-level CF deletion times, + // not the range tombstone. For that we use the columnIndexer tombstone tracker. if (indexBuilder.tombstoneTracker().isDeleted(reduced)) { indexer.remove(reduced); http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1f38cd2/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/RangeTombstoneTest.java index 3314696,80982cd..ecf06d3 --- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java +++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java @@@ -18,41 -18,29 +18,44 @@@ */ package org.apache.cassandra.db; + import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + import java.util.concurrent.ExecutionException; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; ++import org.apache.cassandra.Util; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.IndexType; import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; +import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy; -import org.apache.cassandra.db.index.*; +import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.composites.CellNames; +import org.apache.cassandra.db.composites.Composites; +import org.apache.cassandra.db.filter.ColumnSlice; +import org.apache.cassandra.db.filter.IDiskAtomFilter; +import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.filter.SliceQueryFilter; +import org.apache.cassandra.db.index.PerColumnSecondaryIndex; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.index.SecondaryIndexSearcher; +import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.SSTableReader; -import org.apache.cassandra.thrift.IndexType; - -import org.junit.Ignore; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.*; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.memory.MemtableAllocator; import static org.apache.cassandra.Util.dk; import static org.junit.Assert.assertEquals; @@@ -224,6 -111,61 +227,59 @@@ public class RangeTombstoneTest extend } @Test + public void test7808_1() throws ExecutionException, InterruptedException + { - DatabaseDescriptor.setInMemoryCompactionLimit(0); + Keyspace ks = Keyspace.open(KSNAME); + ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); + cfs.metadata.gcGraceSeconds(2); + + String key = "7808_1"; - RowMutation rm; - rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); ++ Mutation rm; ++ rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + for (int i = 0; i < 40; i += 2) + add(rm, i, 0); + rm.apply(); + cfs.forceBlockingFlush(); - rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); ++ rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + ColumnFamily cf = rm.addOrGet(CFNAME); + cf.delete(new DeletionInfo(1, 1)); + rm.apply(); + cfs.forceBlockingFlush(); + Thread.sleep(5); + cfs.forceMajorCompaction(); + } + + @Test + public void test7808_2() throws ExecutionException, InterruptedException, IOException + { - DatabaseDescriptor.setInMemoryCompactionLimit(0); + Keyspace ks = Keyspace.open(KSNAME); + ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME); + cfs.metadata.gcGraceSeconds(2); + + String key = "7808_2"; - RowMutation rm; - rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); ++ Mutation rm; ++ rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + for (int i = 10; i < 20; i++) + add(rm, i, 0); + rm.apply(); + cfs.forceBlockingFlush(); + - rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); ++ rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + ColumnFamily cf = rm.addOrGet(CFNAME); + cf.delete(new DeletionInfo(0,0)); + rm.apply(); + - rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); ++ rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key)); + add(rm, 5, 1); + rm.apply(); + + cfs.forceBlockingFlush(); + Thread.sleep(5); + cfs.forceMajorCompaction(); + assertEquals(1, Util.getColumnFamily(ks, Util.dk(key), CFNAME).getColumnCount()); + } + + @Test public void overlappingRangeTest() throws Exception { CompactionManager.instance.disableAutoCompaction();