Merge branch 'cassandra-2.2' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9ae7126d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9ae7126d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9ae7126d Branch: refs/heads/trunk Commit: 9ae7126d3043071e02fc413c65bfbb092c852276 Parents: ece5cfc fb4656f Author: Yuki Morishita <yu...@apache.org> Authored: Tue Jul 21 23:18:59 2015 -0500 Committer: Yuki Morishita <yu...@apache.org> Committed: Tue Jul 21 23:18:59 2015 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 55 ++++++++++++++++-- .../org/apache/cassandra/db/Directories.java | 61 +++++++++++++++----- .../repair/RepairMessageVerbHandler.java | 3 +- .../cassandra/db/ColumnFamilyStoreTest.java | 40 ++++++++++++- .../unit/org/apache/cassandra/db/ScrubTest.java | 2 +- 6 files changed, 138 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ae7126d/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 2a3aecd,e1d1fba..3769f83 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,29 -1,7 +1,30 @@@ +3.0 + * Allow extra schema definitions in cassandra-stress yaml (CASSANDRA-9850) + * Metrics should use up to date nomenclature (CASSANDRA-9448) + * Change CREATE/ALTER TABLE syntax for compression (CASSANDRA-8384) + * Cleanup crc and adler code for java 8 (CASSANDRA-9650) + * Storage engine refactor (CASSANDRA-8099, 9743, 9746, 9759, 9781, 9808, 9825, 9848) + * Update Guava to 18.0 (CASSANDRA-9653) + * Bloom filter false positive ratio is not honoured (CASSANDRA-8413) + * New option for cassandra-stress to leave a ratio of columns null (CASSANDRA-9522) + * Change hinted_handoff_enabled yaml setting, JMX (CASSANDRA-9035) + * Add algorithmic token allocation (CASSANDRA-7032) + * Add nodetool command to replay batchlog (CASSANDRA-9547) + * Make file buffer cache independent of paths being read (CASSANDRA-8897) + * Remove deprecated legacy Hadoop code (CASSANDRA-9353) + * Decommissioned nodes will not rejoin the cluster (CASSANDRA-8801) + * Change gossip stabilization to use endpoit size (CASSANDRA-9401) + * Change default garbage collector to G1 (CASSANDRA-7486) + * Populate TokenMetadata early during startup (CASSANDRA-9317) + * undeprecate cache recentHitRate (CASSANDRA-6591) + * Add support for selectively varint encoding fields (CASSANDRA-9499) + + 2.2.1 * UDF / UDA execution time in trace (CASSANDRA-9723) + * Remove repair snapshot leftover on startup (CASSANDRA-7357) + 2.2.0 * Fix cqlsh copy methods and other windows specific issues (CASSANDRA-9795) * Don't wrap byte arrays in SequentialWriter (CASSANDRA-9797) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ae7126d/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index f74fd3c,7d52a94..d3ad4e6 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -39,8 -40,6 +40,7 @@@ import org.apache.cassandra.db.lifecycl import org.apache.cassandra.db.lifecycle.Tracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.metrics.TableMetrics; - import org.apache.cassandra.utils.memory.MemtablePool; import org.json.simple.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ae7126d/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 3620bd0,b5e62b3..deb1480 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@@ -28,60 -54,119 +28,63 @@@ import org.junit.BeforeClass import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.OrderedJUnit4ClassRunner; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.Iterators; +import org.apache.cassandra.*; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.db.marshal.LexicalUUIDType; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.Bounds; -import org.apache.cassandra.dht.ExcludingBounds; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.IncludingExcludingBounds; -import org.apache.cassandra.dht.Range; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.ClearableHistogram; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -import org.apache.cassandra.thrift.ThriftValidation; +import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; + import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.WrappedRunnable; -import org.apache.thrift.TException; - -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.dk; -import static org.apache.cassandra.Util.rp; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; @RunWith(OrderedJUnit4ClassRunner.class) public class ColumnFamilyStoreTest { - static byte[] bytes1, bytes2; public static final String KEYSPACE1 = "ColumnFamilyStoreTest1"; public static final String KEYSPACE2 = "ColumnFamilyStoreTest2"; - public static final String KEYSPACE3 = "ColumnFamilyStoreTest3"; - public static final String KEYSPACE4 = "PerRowSecondaryIndex"; public static final String CF_STANDARD1 = "Standard1"; public static final String CF_STANDARD2 = "Standard2"; - public static final String CF_STANDARD3 = "Standard3"; - public static final String CF_STANDARD4 = "Standard4"; - public static final String CF_STANDARD5 = "Standard5"; - public static final String CF_STANDARDINT = "StandardInteger1"; public static final String CF_SUPER1 = "Super1"; public static final String CF_SUPER6 = "Super6"; + public static final String CF_INDEX1 = "Indexed1"; - public static final String CF_INDEX2 = "Indexed2"; - public static final String CF_INDEX3 = "Indexed3"; - - static - { - Random random = new Random(); - bytes1 = new byte[1024]; - bytes2 = new byte[128]; - random.nextBytes(bytes1); - random.nextBytes(bytes2); - } @BeforeClass public static void defineSchema() throws ConfigurationException { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), + KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2)); + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD5), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX1, true), - SchemaLoader.indexCFMD(KEYSPACE1, CF_INDEX2, false), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance), - SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER6, LexicalUUIDType.instance, UTF8Type.instance), - SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDINT, IntegerType.instance)); ++ SchemaLoader.keysIndexCFMD(KEYSPACE1, CF_INDEX1, true)); + // TODO: Fix superCFMD failing on legacy table creation. Seems to be applying composite comparator to partition key + // SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, LongType.instance)); + // SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER6, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", LexicalUUIDType.instance, UTF8Type.instance), SchemaLoader.createKeyspace(KEYSPACE2, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1), - SchemaLoader.indexCFMD(KEYSPACE2, CF_INDEX1, true), - SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEX2, true), - SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEX3, true).gcGraceSeconds(0)); - SchemaLoader.createKeyspace(KEYSPACE3, - SimpleStrategy.class, - KSMetaData.optsWithRF(5), - SchemaLoader.indexCFMD(KEYSPACE3, CF_INDEX1, true)); - SchemaLoader.createKeyspace(KEYSPACE4, - SimpleStrategy.class, - KSMetaData.optsWithRF(1), - SchemaLoader.perRowIndexedCFMD(KEYSPACE4, "Indexed1")); + KeyspaceParams.simple(1), + SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1)); + } + + @Before + public void truncateCFS() + { + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking(); + // Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_SUPER1).truncateBlocking(); + + Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); } @Test @@@ -291,17 -1034,30 +294,52 @@@ // and it remains so after flush. (this wasn't failing before, but it's good to check.) cfs.forceBlockingFlush(); - assertRowAndColCount(1, 2, true, cfs.getRangeSlice(Util.range("f", "g"), null, ThriftValidation.asIFilter(sp, cfs.metadata, null), 100)); + assertRangeCount(cfs, col, val, 4); } - - private ColumnFamilyStore insertKey1Key2() + @Test ++ public void testClearEphemeralSnapshots() throws Throwable + { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); - List<Mutation> rms = new LinkedList<>(); - Mutation rm; - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("key1")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rms.add(rm); - Util.writeColumnFamily(rms); ++ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1); ++ ++ //cleanup any previous test gargbage ++ cfs.clearSnapshot(""); + - rm = new Mutation(KEYSPACE2, ByteBufferUtil.bytes("key2")); - rm.add(CF_STANDARD1, cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); - rms.add(rm); - return Util.writeColumnFamily(rms); ++ int numRows = 1000; ++ long[] colValues = new long [numRows * 2]; // each row has two columns ++ for (int i = 0; i < colValues.length; i+=2) ++ { ++ colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column ++ colValues[i+1] = 3L; //other column ++ } ++ ScrubTest.fillIndexCF(cfs, false, colValues); ++ ++ cfs.snapshot("nonEphemeralSnapshot", null, false); ++ cfs.snapshot("ephemeralSnapshot", null, true); ++ ++ Map<String, Pair<Long, Long>> snapshotDetails = cfs.getSnapshotDetails(); ++ assertEquals(2, snapshotDetails.size()); ++ assertTrue(snapshotDetails.containsKey("ephemeralSnapshot")); ++ assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot")); ++ ++ ColumnFamilyStore.clearEphemeralSnapshots(cfs.directories); ++ ++ snapshotDetails = cfs.getSnapshotDetails(); ++ assertEquals(1, snapshotDetails.size()); ++ assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot")); ++ ++ //test cleanup ++ cfs.clearSnapshot(""); + } + + @Test public void testBackupAfterFlush() throws Throwable { - ColumnFamilyStore cfs = insertKey1Key2(); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); + cfs.forceBlockingFlush(); + new RowUpdateBuilder(cfs.metadata, 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); + cfs.forceBlockingFlush(); for (int version = 1; version <= 2; ++version) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/9ae7126d/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java index 2574847,0d90354..6f20ccf --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@@ -446,7 -485,7 +446,7 @@@ public class ScrubTes cfs.forceBlockingFlush(); } -- private void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long ... values) ++ public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long ... values) { assertTrue(values.length % 2 == 0); for (int i = 0; i < values.length; i +=2)
