Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3128802a Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3128802a Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3128802a Branch: refs/heads/cassandra-3.0 Commit: 3128802a5f9d83a2f98794edd7f94890dd070a90 Parents: 1d7c0bc cc75de6 Author: Marcus Eriksson <marc...@apache.org> Authored: Tue Apr 26 08:25:45 2016 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Tue Apr 26 08:25:45 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../compaction/LeveledCompactionStrategy.java | 15 ++++- .../db/compaction/LeveledManifest.java | 15 ++++- .../LongLeveledCompactionStrategyTest.java | 60 ++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/3128802a/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 1837a6e,d170def..06b27b4 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -8,61 -3,18 +8,62 @@@ Merged from 2.1 * cqlsh COPY FROM fails with []{} chars in UDT/tuple fields/values (CASSANDRA-11633) * clqsh: COPY FROM throws TypeError with Cython extensions enabled (CASSANDRA-11574) * cqlsh: COPY FROM ignores NULL values in conversion (CASSANDRA-11549) + * (cqlsh) Fix potential COPY deadlock when parent process is terminating child + processes (CASSANDRA-11505) + * Validate levels when building LeveledScanner to avoid overlaps with orphaned sstables (CASSANDRA-9935) -2.1.14 - * (cqlsh) Fix potential COPY deadlock when parent process is terminating child - processes (CASSANDRA-11505) - * Replace sstables on DataTracker before marking them as non-compacting during anti-compaction (CASSANDRA-11548) +2.2.6 + * Allow only DISTINCT queries with partition keys restrictions (CASSANDRA-11339) + * CqlConfigHelper no longer requires both a keystore and truststore to work (CASSANDRA-11532) + * Make deprecated repair methods backward-compatible with previous notification service (CASSANDRA-11430) + * IncomingStreamingConnection version check message wrong (CASSANDRA-11462) + * DatabaseDescriptor should log stacktrace in case of Eception during seed provider creation (CASSANDRA-11312) + * Use canonical path for directory in SSTable descriptor (CASSANDRA-10587) + * Add cassandra-stress keystore option (CASSANDRA-9325) + * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448). + * Dont mark sstables as repairing with sub range repairs (CASSANDRA-11451) + * Fix use of NullUpdater for 2i during compaction (CASSANDRA-11450) + * Notify when sstables change after cancelling compaction (CASSANDRA-11373) + * cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333) + * Add -Dcassandra.start_gossip startup option (CASSANDRA-10809) + * Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748) + * Clarify that now() function is calculated on the coordinator node in CQL documentation (CASSANDRA-10900) + * Fix bloom filter sizing with LCS (CASSANDRA-11344) + * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092) + * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196) + * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301) + * Add missing newline at end of bin/cqlsh (CASSANDRA-11325) + * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297) + * Unresolved hostname leads to replace being ignored (CASSANDRA-11210) + * Fix filtering on non-primary key columns for thrift static column families + (CASSANDRA-6377) + * Only log yaml config once, at startup (CASSANDRA-11217) + * Preserve order for preferred SSL cipher suites (CASSANDRA-11164) + * Reference leak with parallel repairs on the same table (CASSANDRA-11215) + * Range.compareTo() violates the contract of Comparable (CASSANDRA-11216) + * Avoid NPE when serializing ErrorMessage with null message (CASSANDRA-11167) + * Replacing an aggregate with a new version doesn't reset INITCOND (CASSANDRA-10840) + * (cqlsh) cqlsh cannot be called through symlink (CASSANDRA-11037) + * fix ohc and java-driver pom dependencies in build.xml (CASSANDRA-10793) + * Protect from keyspace dropped during repair (CASSANDRA-11065) + * Handle adding fields to a UDT in SELECT JSON and toJson() (CASSANDRA-11146) + * Better error message for cleanup (CASSANDRA-10991) + * cqlsh pg-style-strings broken if line ends with ';' (CASSANDRA-11123) + * Use cloned TokenMetadata in size estimates to avoid race against membership check + (CASSANDRA-10736) + * Always persist upsampled index summaries (CASSANDRA-10512) + * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733) + * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048) + * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281) + * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030) + * Fix paging on DISTINCT queries repeats result when first row in partition changes + (CASSANDRA-10010) +Merged from 2.1: * Checking if an unlogged batch is local is inefficient (CASSANDRA-11529) * Fix paging for COMPACT tables without clustering columns (CASSANDRA-11467) - * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448) - * Backport CASSANDRA-10859 (CASSANDRA-11415) - * COPY FROM fails when importing blob (CASSANDRA-11375) + * Add a -j parameter to scrub/cleanup/upgradesstables to state how + many threads to use (CASSANDRA-11179) * Backport CASSANDRA-10679 (CASSANDRA-9598) * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342) * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286) http://git-wip-us.apache.org/repos/asf/cassandra/blob/3128802a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/3128802a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 544b0a8,11af6c4..3b207da --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@@ -23,11 -23,12 +23,12 @@@ import java.util.* import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; + import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/cassandra/blob/3128802a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java ---------------------------------------------------------------------- diff --cc test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java index e38eb3c,b3dfb3c..5439a72 --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java @@@ -18,11 -18,14 +18,13 @@@ package org.apache.cassandra.db.compaction; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.*; ++import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; ++import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; @@@ -138,4 -140,152 +140,62 @@@ public class LongLeveledCompactionStrat } } } + - class CheckThatSSTableIsReleasedOnlyAfterCompactionFinishes implements INotificationConsumer - { - public final Set<SSTableReader> finishedCompaction = new HashSet<>(); - - boolean failed = false; - - public void handleNotification(INotification received, Object sender) - { - if (received instanceof SSTableCompactingNotification) - { - SSTableCompactingNotification notification = (SSTableCompactingNotification) received; - if (!notification.compacting) - { - for (SSTableReader reader : notification.sstables) - { - finishedCompaction.add(reader); - } - } - } - if (received instanceof SSTableListChangedNotification) - { - SSTableListChangedNotification notification = (SSTableListChangedNotification) received; - for (SSTableReader reader : notification.removed) - { - if (finishedCompaction.contains(reader)) - failed = true; - } - } - } - - boolean isFailed() - { - return failed; - } - } - - @Test - public void testAntiCompactionAfterLCS() throws Exception - { - testParallelLeveledCompaction(); - - String ksname = "Keyspace1"; - String cfname = "StandardLeveled"; - Keyspace keyspace = Keyspace.open(ksname); - ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname); - WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy()); - - Collection<SSTableReader> initialSSTables = store.getUnrepairedSSTables(); - assertEquals(store.getSSTables().size(), initialSSTables.size()); - - CheckThatSSTableIsReleasedOnlyAfterCompactionFinishes checker = new CheckThatSSTableIsReleasedOnlyAfterCompactionFinishes(); - store.getDataTracker().subscribe(checker); - - //anti-compact a subset of sstables - Range<Token> range = new Range<Token>(new BytesToken("110".getBytes()), new BytesToken("111".getBytes()), store.partitioner); - List<Range<Token>> ranges = Arrays.asList(range); - Refs<SSTableReader> refs = Refs.tryRef(initialSSTables); - if (refs == null) - throw new IllegalStateException(); - long repairedAt = 1000; - CompactionManager.instance.performAnticompaction(store, ranges, refs, repairedAt); - - //check that sstables were released only after compaction finished - assertFalse("Anti-compaction released sstable from compacting set before compaction was finished", - checker.isFailed()); - - //check there is only one global ref count - for (SSTableReader sstable : store.getSSTables()) - { - assertFalse(sstable.isMarkedCompacted()); - assertEquals(1, sstable.selfRef().globalCount()); - } - - //check that compacting status was clearedd in all sstables - assertEquals(0, store.getDataTracker().getCompacting().size()); - - //make sure readers were replaced correctly on unrepaired leveled manifest after anti-compaction - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1); - for (SSTableReader reader : initialSSTables) - { - Range<Token> sstableRange = new Range<Token>(reader.first.getToken(), reader.last.getToken()); - if (sstableRange.intersects(range)) - { - assertFalse(lcs.manifest.generations[reader.getSSTableLevel()].contains(reader)); - } - } - } - + @Test + public void testLeveledScanner() throws Exception + { + testParallelLeveledCompaction(); - String ksname = "Keyspace1"; - String cfname = "StandardLeveled"; - Keyspace keyspace = Keyspace.open(ksname); - ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname); ++ Keyspace keyspace = Keyspace.open(KEYSPACE1); ++ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL); + store.disableAutoCompaction(); + + WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy()); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1); + + ByteBuffer value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value + + // Adds 10 partitions + for (int r = 0; r < 10; r++) + { + DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(ksname, key.getKey()); ++ Mutation rm = new Mutation(KEYSPACE1, key.getKey()); + for (int c = 0; c < 10; c++) + { - rm.add(cfname, Util.cellname("column" + c), value, 0); ++ rm.add(CF_STANDARDLVL, Util.cellname("column" + c), value, 0); + } + rm.apply(); + } + + //Flush sstable + store.forceBlockingFlush(); + + Collection<SSTableReader> allSSTables = store.getSSTables(); + for (SSTableReader sstable : allSSTables) + { + if (sstable.getSSTableLevel() == 0) + { + System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename()); + sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1); + sstable.reloadSSTableMetadata(); + } + } + + try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables)) + { + //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935) + for (ISSTableScanner scanner : scannerList.scanners) + { + DecoratedKey lastKey = null; + while (scanner.hasNext()) + { + OnDiskAtomIterator row = scanner.next(); + if (lastKey != null) + { + assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0); + } + lastKey = row.getKey(); + } + } + } + } }
