Merge branch 'cassandra-1.2' into cassandra-2.0 Conflicts: src/java/org/apache/cassandra/thrift/CassandraServer.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b4337228 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b4337228 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b4337228 Branch: refs/heads/cassandra-2.1 Commit: b4337228df7178183a643a8f201e9389cab36ab3 Parents: 90d086a 1052749 Author: Sylvain Lebresne <sylv...@datastax.com> Authored: Wed Apr 30 11:24:21 2014 +0200 Committer: Sylvain Lebresne <sylv...@datastax.com> Committed: Wed Apr 30 11:24:21 2014 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/config/CFMetaData.java | 47 +++++++++++++++++--- .../cassandra/thrift/CassandraServer.java | 3 +- .../unit/org/apache/cassandra/SchemaLoader.java | 16 ++++++- .../apache/cassandra/config/CFMetaDataTest.java | 32 ++++--------- 5 files changed, 65 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/b4337228/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 428f8fc,fa9a156..e25e71f --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -23,64 -11,13 +23,65 @@@ Merged from 1.2 * Fix CQLSH parsing of functions and BLOB literals (CASSANDRA-7018) * Require nodetool rebuild_index to specify index names (CASSANDRA-7038) * Ensure that batchlog and hint timeouts do not produce hints (CASSANDRA-7058) - * Don't shut MessagingService down when replacing a node (CASSANDRA-6476) * Always clean up references in SerializingCache (CASSANDRA-6994) + * Don't shut MessagingService down when replacing a node (CASSANDRA-6476) * fix npe when doing -Dcassandra.fd_initial_value_ms (CASSANDRA-6751) + * Preserves CQL metadata when updating table from thrift (CASSANDRA-6831) -1.2.16 +2.0.7 + * Put nodes in hibernate when join_ring is false (CASSANDRA-6961) + * Avoid early loading of non-system keyspaces before compaction-leftovers + cleanup at startup (CASSANDRA-6913) + * Restrict Windows to parallel repairs (CASSANDRA-6907) + * (Hadoop) Allow manually specifying start/end tokens in CFIF (CASSANDRA-6436) + * Fix NPE in MeteredFlusher (CASSANDRA-6820) + * Fix race processing range scan responses (CASSANDRA-6820) + * Allow deleting snapshots from dropped keyspaces (CASSANDRA-6821) + * Add uuid() function (CASSANDRA-6473) + * Omit tombstones from schema digests (CASSANDRA-6862) + * Include correct consistencyLevel in LWT timeout (CASSANDRA-6884) + * Lower chances for losing new SSTables during nodetool refresh and + ColumnFamilyStore.loadNewSSTables (CASSANDRA-6514) + * Add support for DELETE ... IF EXISTS to CQL3 (CASSANDRA-5708) + * Update hadoop_cql3_word_count example (CASSANDRA-6793) + * Fix handling of RejectedExecution in sync Thrift server (CASSANDRA-6788) + * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865) + * Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864) + * Fix schema concurrency exceptions (CASSANDRA-6841) + * Fix leaking validator FH in StreamWriter (CASSANDRA-6832) + * Fix saving triggers to schema (CASSANDRA-6789) + * Fix trigger mutations when base mutation list is immutable (CASSANDRA-6790) + * Fix accounting in FileCacheService to allow re-using RAR (CASSANDRA-6838) + * Fix static counter columns (CASSANDRA-6827) + * Restore expiring->deleted (cell) compaction optimization (CASSANDRA-6844) + * Fix CompactionManager.needsCleanup (CASSANDRA-6845) + * Correctly compare BooleanType values other than 0 and 1 (CASSANDRA-6779) + * Read message id as string from earlier versions (CASSANDRA-6840) + * Properly use the Paxos consistency for (non-protocol) batch (CASSANDRA-6837) + * Add paranoid disk failure option (CASSANDRA-6646) + * Improve PerRowSecondaryIndex performance (CASSANDRA-6876) + * Extend triggers to support CAS updates (CASSANDRA-6882) + * Static columns with IF NOT EXISTS don't always work as expected (CASSANDRA-6873) + * Fix paging with SELECT DISTINCT (CASSANDRA-6857) + * Fix UnsupportedOperationException on CAS timeout (CASSANDRA-6923) + * Improve MeteredFlusher handling of MF-unaffected column families + (CASSANDRA-6867) + * Add CqlRecordReader using native pagination (CASSANDRA-6311) + * Add QueryHandler interface (CASSANDRA-6659) + * Track liveRatio per-memtable, not per-CF (CASSANDRA-6945) + * Make sure upgradesstables keeps sstable level (CASSANDRA-6958) + * Fix LIMIT with static columns (CASSANDRA-6956) + * Fix clash with CQL column name in thrift validation (CASSANDRA-6892) + * Fix error with super columns in mixed 1.2-2.0 clusters (CASSANDRA-6966) + * Fix bad skip of sstables on slice query with composite start/finish (CASSANDRA-6825) + * Fix unintended update with conditional statement (CASSANDRA-6893) + * Fix map element access in IF (CASSANDRA-6914) + * Avoid costly range calculations for range queries on system keyspaces + (CASSANDRA-6906) + * Fix SSTable not released if stream session fails (CASSANDRA-6818) + * Avoid build failure due to ANTLR timeout (CASSANDRA-6991) +Merged from 1.2: * Add UNLOGGED, COUNTER options to BATCH documentation (CASSANDRA-6816) * add extra SSL cipher suites (CASSANDRA-6613) * fix nodetool getsstables for blob PK (CASSANDRA-6803) http://git-wip-us.apache.org/repos/asf/cassandra/blob/b4337228/src/java/org/apache/cassandra/config/CFMetaData.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/config/CFMetaData.java index 23f912e,9e3ceb7..9ca41ac --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@@ -863,7 -668,7 +863,7 @@@ public final class CFMetaDat } /** applies implicit defaults to cf definition. useful in updates */ -- public static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) ++ private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def) { if (!cf_def.isSetComment()) cf_def.setComment(""); @@@ -895,10 -698,12 +895,48 @@@ public static CFMetaData fromThrift(org.apache.cassandra.thrift.CfDef cf_def) throws InvalidRequestException, ConfigurationException { - ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); - if (cfType == null) ++ CFMetaData cfm = internalFromThrift(cf_def); ++ ++ if (cf_def.isSetKey_alias() && !(cfm.keyValidator instanceof CompositeType)) ++ cfm.column_metadata.put(cf_def.key_alias, ColumnDefinition.partitionKeyDef(cf_def.key_alias, cfm.keyValidator, null)); ++ ++ return cfm.rebuild(); ++ } ++ ++ public static CFMetaData fromThriftForUpdate(org.apache.cassandra.thrift.CfDef cf_def, CFMetaData toUpdate) throws InvalidRequestException, ConfigurationException ++ { ++ CFMetaData cfm = internalFromThrift(cf_def); ++ ++ // Thrift update can't have CQL metadata, and so we'll copy the ones of the updated metadata (to make ++ // sure we don't override anything existing -- see #6831). One exception (for historical reasons) is ++ // the partition key column name however, which can be provided through thrift. If it is, make sure ++ // we use the one of the update. ++ boolean hasKeyAlias = cf_def.isSetKey_alias() && !(cfm.keyValidator instanceof CompositeType); ++ if (hasKeyAlias) ++ cfm.column_metadata.put(cf_def.key_alias, ColumnDefinition.partitionKeyDef(cf_def.key_alias, cfm.keyValidator, null)); ++ ++ for (ColumnDefinition def : toUpdate.allColumns()) + { - throw new InvalidRequestException("Invalid column type " + cf_def.column_type); ++ // isPartOfCellName basically means 'is not just a CQL metadata' ++ if (def.isPartOfCellName()) ++ continue; ++ ++ if (def.type == ColumnDefinition.Type.PARTITION_KEY && hasKeyAlias) ++ continue; ++ ++ cfm.addOrReplaceColumnDefinition(def); + } + ++ return cfm.rebuild(); ++ } ++ ++ // Do most of the work, but don't handle CQL metadata (i.e. skip key_alias and don't rebuild()) ++ private static CFMetaData internalFromThrift(org.apache.cassandra.thrift.CfDef cf_def) throws InvalidRequestException, ConfigurationException ++ { + ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type); + if (cfType == null) + throw new InvalidRequestException("Invalid column type " + cf_def.column_type); + applyImplicitDefaults(cf_def); try @@@ -939,20 -736,14 +977,15 @@@ CompressionParameters cp = CompressionParameters.create(cf_def.compression_options); + if (cf_def.isSetKey_validation_class()) { newCFMD.keyValidator(TypeParser.parse(cf_def.key_validation_class)); } - if (cf_def.isSetKey_alias() && !(newCFMD.keyValidator instanceof CompositeType)) - { - newCFMD.column_metadata.put(cf_def.key_alias, ColumnDefinition.partitionKeyDef(cf_def.key_alias, newCFMD.keyValidator, null)); - } + return newCFMD.comment(cf_def.comment) .replicateOnWrite(cf_def.replicate_on_write) .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) - .keyValidator(TypeParser.parse(cf_def.key_validation_class)) - .columnMetadata(ColumnDefinition.fromThrift(cf_def.column_metadata)) + .columnMetadata(ColumnDefinition.fromThrift(cf_def.column_metadata, newCFMD.isSuper())) - .compressionParameters(cp) - .rebuild(); + .compressionParameters(cp); } - catch (SyntaxException e) + catch (SyntaxException | MarshalException e) { throw new ConfigurationException(e.getMessage()); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/b4337228/src/java/org/apache/cassandra/thrift/CassandraServer.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/thrift/CassandraServer.java index 8a8c40e,588f732..d47d988 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@@ -1648,18 -1420,26 +1648,17 @@@ public class CassandraServer implement if (oldCfm == null) throw new InvalidRequestException("Could not find column family definition to modify."); - if (oldCfm.isThriftIncompatible()) + if (!oldCfm.isThriftCompatible()) throw new InvalidRequestException("Cannot modify CQL3 table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify CQL3 tables instead."); - CFMetaData.applyImplicitDefaults(cf_def); - CFMetaData cfm = CFMetaData.fromThrift(cf_def); - state().hasColumnFamilyAccess(cf_def.keyspace, cf_def.name, Permission.ALTER); - - CFMetaData.applyImplicitDefaults(cf_def); - CFMetaData cfm = CFMetaData.fromThrift(cf_def); ++ CFMetaData cfm = CFMetaData.fromThriftForUpdate(cf_def, oldCfm); + CFMetaData.validateCompactionOptions(cfm.compactionStrategyClass, cfm.compactionStrategyOptions); + cfm.addDefaultIndexNames(); - /* - * CASSANDRA-6831: Because thrift updates don't know about aliases, - * we should copy them from the original CFM - */ - if (!cf_def.isSetKey_alias()) - cfm.keyAliases(oldCfm.getKeyAliases()); - cfm.columnAliases(oldCfm.getColumnAliases()); - cfm.valueAlias(oldCfm.getValueAlias()); + if (!oldCfm.getTriggers().equals(cfm.getTriggers())) + state().ensureIsSuper("Only superusers are allowed to add or remove triggers."); - CFMetaData.validateCompactionOptions(cfm.compactionStrategyClass, cfm.compactionStrategyOptions, false); - cfm.addDefaultIndexNames(); - MigrationManager.announceColumnFamilyUpdate(cfm); + MigrationManager.announceColumnFamilyUpdate(cfm, true); return Schema.instance.getVersion().toString(); } catch (RequestValidationException e) http://git-wip-us.apache.org/repos/asf/cassandra/blob/b4337228/test/unit/org/apache/cassandra/SchemaLoader.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/SchemaLoader.java index daff0de,b2ae83a..e4929ee --- a/test/unit/org/apache/cassandra/SchemaLoader.java +++ b/test/unit/org/apache/cassandra/SchemaLoader.java @@@ -307,25 -303,6 +307,39 @@@ public class SchemaLoade opts_rf1, perRowIndexedCFMD(ks_prsi, "Indexed1", withOldCfIds))); + // CQLKeyspace + schema.add(KSMetaData.testMetadata(ks_cql, + simple, + opts_rf1, + + // Column Families + CFMetaData.compile("CREATE TABLE table1 (" + + "k int PRIMARY KEY," + + "v1 text," + + "v2 int" + + ")", ks_cql), + + CFMetaData.compile("CREATE TABLE table2 (" + + "k text," + + "c text," + + "v text," - + "PRIMARY KEY (k, c))", ks_cql) ++ + "PRIMARY KEY (k, c))", ks_cql), ++ CFMetaData.compile("CREATE TABLE foo (" ++ + "bar text, " ++ + "baz text, " ++ + "qux text, " ++ + "PRIMARY KEY(bar, baz) ) " ++ + "WITH COMPACT STORAGE", ks_cql), ++ CFMetaData.compile("CREATE TABLE foofoo (" ++ + "bar text, " ++ + "baz text, " ++ + "qux text, " ++ + "quz text, " ++ + "foo text, " ++ + "PRIMARY KEY((bar, baz), qux, quz) ) " ++ + "WITH COMPACT STORAGE", ks_cql) + )); + if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))) useCompression(schema); http://git-wip-us.apache.org/repos/asf/cassandra/blob/b4337228/test/unit/org/apache/cassandra/config/CFMetaDataTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/config/CFMetaDataTest.java index b8c061f,8ca83ce..b0cafc4 --- a/test/unit/org/apache/cassandra/config/CFMetaDataTest.java +++ b/test/unit/org/apache/cassandra/config/CFMetaDataTest.java @@@ -18,10 -18,11 +18,9 @@@ */ package org.apache.cassandra.config; -import java.nio.ByteBuffer; import java.util.ArrayList; - import java.util.Iterator; import java.util.List; import java.util.HashMap; -import java.util.Map; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.cql3.QueryProcessor; @@@ -35,10 -42,10 +34,11 @@@ import org.apache.cassandra.thrift.CfDe import org.apache.cassandra.thrift.ColumnDef; import org.apache.cassandra.thrift.IndexType; import org.apache.cassandra.utils.ByteBufferUtil; -- import org.junit.Test; ++import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; ++import static org.junit.Assert.assertThat; public class CFMetaDataTest extends SchemaLoader { @@@ -98,11 -106,11 +98,13 @@@ @Test public void testConversionsInverses() throws Exception { - for (String table : Schema.instance.getNonSystemTables()) + for (String keyspaceName : Schema.instance.getNonSystemKeyspaces()) { - for (ColumnFamilyStore cfs : Table.open(table).getColumnFamilyStores()) + for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores()) { CFMetaData cfm = cfs.metadata; ++ if (!cfm.isThriftCompatible()) ++ continue; checkInverses(cfm); // Testing with compression to catch #3558 @@@ -135,17 -125,27 +119,17 @@@ { DecoratedKey k = StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(cfm.ksName)); - // This is a nasty hack to work around the fact that non-null componentIndex - // are only used by CQL (so far) so we don't expose them through thrift - // There is a CFM with componentIndex defined in Keyspace2 which is used by - // ColumnFamilyStoreTest to verify index repair (CASSANDRA-2897) - for (Map.Entry<ByteBuffer, ColumnDefinition> cMeta: cfm.column_metadata.entrySet()) - { - // Non-null componentIndex are only used by CQL (so far) so we don't expose - // them through thrift - if (cMeta.getValue().componentIndex != null) - cfm.column_metadata.remove(cMeta.getKey()); - } - // Test thrift conversion - CFMetaData before = withoutThriftIncompatible(cfm); - CFMetaData after = withoutThriftIncompatible(CFMetaData.fromThrift(before.toThrift())); - assert before.equals(after) : String.format("\n%s\n!=\n%s", before, after); - assert cfm.equals(CFMetaData.fromThrift(cfm.toThrift())) : String.format("\n%s\n!=\n%s", cfm, CFMetaData.fromThrift(cfm.toThrift())); ++ CFMetaData before = cfm; ++ CFMetaData after = CFMetaData.fromThriftForUpdate(before.toThrift(), before); ++ assertThat(after, is(before)); // Test schema conversion RowMutation rm = cfm.toSchema(System.currentTimeMillis()); - ColumnFamily serializedCf = rm.getColumnFamily(Schema.instance.getId(Table.SYSTEM_KS, SystemTable.SCHEMA_COLUMNFAMILIES_CF)); - ColumnFamily serializedCD = rm.getColumnFamily(Schema.instance.getId(Table.SYSTEM_KS, SystemTable.SCHEMA_COLUMNS_CF)); + ColumnFamily serializedCf = rm.getColumnFamily(Schema.instance.getId(Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF)); + ColumnFamily serializedCD = rm.getColumnFamily(Schema.instance.getId(Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNS_CF)); UntypedResultSet.Row result = QueryProcessor.resultify("SELECT * FROM system.schema_columnfamilies", new Row(k, serializedCf)).one(); - CFMetaData newCfm = CFMetaData.addColumnDefinitionSchema(CFMetaData.fromSchemaNoColumns(result), new Row(k, serializedCD)); - assert cfm.equals(newCfm) : String.format("\n%s\n!=\n%s", cfm, newCfm); + CFMetaData newCfm = CFMetaData.addColumnDefinitionsFromSchema(CFMetaData.fromSchemaNoColumnsNoTriggers(result), new Row(k, serializedCD)); - assert cfm.equals(newCfm) : String.format("\n%s\n!=\n%s", cfm, newCfm); ++ assertThat(newCfm, is(cfm)); } }