Merge branch 'cassandra-2.0' into cassandra-2.1 Conflicts: CHANGES.txt src/java/org/apache/cassandra/db/DefsTables.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/42582ecd Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/42582ecd Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/42582ecd Branch: refs/heads/cassandra-2.1 Commit: 42582ecd4bb69998ecd664222012cf5cfef377e0 Parents: e711929 0393c30 Author: Aleksey Yeschenko <alek...@apache.org> Authored: Fri Oct 10 20:02:30 2014 +0300 Committer: Aleksey Yeschenko <alek...@apache.org> Committed: Fri Oct 10 20:02:30 2014 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/DefsTables.java | 261 +++++++++---------- 2 files changed, 119 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/42582ecd/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 5666a9d,b633e48..61dec54 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,75 -1,5 +1,76 @@@ -2.0.11: +2.1.1 + * Fix EXECUTE request with skipMetadata=false returning no metadata + (CASSANDRA-8054) + * Allow concurrent use of CQLBulkOutputFormat (CASSANDRA-7776) + * Shutdown JVM on OOM (CASSANDRA-7507) + * Upgrade netty version and enable epoll event loop (CASSANDRA-7761) + * Don't duplicate sstables smaller than split size when using + the sstablesplitter tool (CASSANDRA-7616) + * Avoid re-parsing already prepared statements (CASSANDRA-7923) + * Fix some Thrift slice deletions and updates of COMPACT STORAGE + tables with some clustering columns omitted (CASSANDRA-7990) + * Fix filtering for CONTAINS on sets (CASSANDRA-8033) + * Properly track added size (CASSANDRA-7239) + * Allow compilation in java 8 (CASSANDRA-7208) + * Fix Assertion error on RangeTombstoneList diff (CASSANDRA-8013) + * Release references to overlapping sstables during compaction (CASSANDRA-7819) + * Send notification when opening compaction results early (CASSANDRA-8034) + * Make native server start block until properly bound (CASSANDRA-7885) + * (cqlsh) Fix IPv6 support (CASSANDRA-7988) + * Ignore fat clients when checking for endpoint collision (CASSANDRA-7939) + * Make sstablerepairedset take a list of files (CASSANDRA-7995) + * (cqlsh) Tab completeion for indexes on map keys (CASSANDRA-7972) + * (cqlsh) Fix UDT field selection in select clause (CASSANDRA-7891) + * Fix resource leak in event of corrupt sstable + * (cqlsh) Add command line option for cqlshrc file path (CASSANDRA-7131) + * Provide visibility into prepared statements churn (CASSANDRA-7921, CASSANDRA-7930) + * Invalidate prepared statements when their keyspace or table is + dropped (CASSANDRA-7566) + * cassandra-stress: fix support for NetworkTopologyStrategy (CASSANDRA-7945) + * Fix saving caches when a table is dropped (CASSANDRA-7784) + * Add better error checking of new stress profile (CASSANDRA-7716) + * Use ThreadLocalRandom and remove FBUtilities.threadLocalRandom (CASSANDRA-7934) + * Prevent operator mistakes due to simultaneous bootstrap (CASSANDRA-7069) + * cassandra-stress supports whitelist mode for node config (CASSANDRA-7658) + * GCInspector more closely tracks GC; cassandra-stress and nodetool report it (CASSANDRA-7916) + * nodetool won't output bogus ownership info without a keyspace (CASSANDRA-7173) + * Add human readable option to nodetool commands (CASSANDRA-5433) + * Don't try to set repairedAt on old sstables (CASSANDRA-7913) + * Add metrics for tracking PreparedStatement use (CASSANDRA-7719) + * (cqlsh) tab-completion for triggers (CASSANDRA-7824) + * (cqlsh) Support for query paging (CASSANDRA-7514) + * (cqlsh) Show progress of COPY operations (CASSANDRA-7789) + * Add syntax to remove multiple elements from a map (CASSANDRA-6599) + * Support non-equals conditions in lightweight transactions (CASSANDRA-6839) + * Add IF [NOT] EXISTS to create/drop triggers (CASSANDRA-7606) + * (cqlsh) Display the current logged-in user (CASSANDRA-7785) + * (cqlsh) Don't ignore CTRL-C during COPY FROM execution (CASSANDRA-7815) + * (cqlsh) Order UDTs according to cross-type dependencies in DESCRIBE + output (CASSANDRA-7659) + * (cqlsh) Fix handling of CAS statement results (CASSANDRA-7671) + * (cqlsh) COPY TO/FROM improvements (CASSANDRA-7405) + * Support list index operations with conditions (CASSANDRA-7499) + * Add max live/tombstoned cells to nodetool cfstats output (CASSANDRA-7731) + * Validate IPv6 wildcard addresses properly (CASSANDRA-7680) + * (cqlsh) Error when tracing query (CASSANDRA-7613) + * Avoid IOOBE when building SyntaxError message snippet (CASSANDRA-7569) + * SSTableExport uses correct validator to create string representation of partition + keys (CASSANDRA-7498) + * Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689) + * Add support for custom 2i validation (CASSANDRA-7575) + * Pig support for hadoop CqlInputFormat (CASSANDRA-6454) + * Add listen_interface and rpc_interface options (CASSANDRA-7417) + * Improve schema merge performance (CASSANDRA-7444) + * Adjust MT depth based on # of partition validating (CASSANDRA-5263) + * Optimise NativeCell comparisons (CASSANDRA-6755) + * Configurable client timeout for cqlsh (CASSANDRA-7516) + * Include snippet of CQL query near syntax error in messages (CASSANDRA-7111) + * Make repair -pr work with -local (CASSANDRA-7450) + * Fix error in sstableloader with -cph > 1 (CASSANDRA-8007) + * Fix snapshot repair error on indexed tables (CASSANDRA-8020) + * Do not exit nodetool repair when receiving JMX NOTIF_LOST (CASSANDRA-7909) +Merged from 2.0: + * Fix merging schemas with re-dropped keyspaces (CASSANDRA-7256) * Fix counters in supercolumns during live upgrades from 1.2 (CASSANDRA-7188) * Notify DT subscribers when a column family is truncated (CASSANDRA-8088) * Add sanity check of $JAVA on startup (CASSANDRA-7676) http://git-wip-us.apache.org/repos/asf/cassandra/blob/42582ecd/src/java/org/apache/cassandra/db/DefsTables.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/DefsTables.java index fc43c27,35eecc0..59f2e20 --- a/src/java/org/apache/cassandra/db/DefsTables.java +++ b/src/java/org/apache/cassandra/db/DefsTables.java @@@ -122,7 -121,7 +123,7 @@@ public class DefsTable { List<Row> serializedSchema = SystemKeyspace.serializedSchema(SystemKeyspace.SCHEMA_KEYSPACES_CF); -- List<KSMetaData> keyspaces = new ArrayList<KSMetaData>(serializedSchema.size()); ++ List<KSMetaData> keyspaces = new ArrayList<>(serializedSchema.size()); for (Row row : serializedSchema) { @@@ -196,187 -186,108 +197,161 @@@ // it is safe to drop a keyspace only when all nested ColumnFamilies where deleted for (String keyspaceToDrop : keyspacesToDrop) dropKeyspace(keyspaceToDrop); - - Schema.instance.updateVersionAndAnnounce(); } - private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated) + private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily> before, Map<DecoratedKey, ColumnFamily> after) { - // calculate the difference between old and new states (note that entriesOnlyLeft() will be always empty) - MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated); - - /** - * At first step we check if any new keyspaces were added. - */ - for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet()) - { - ColumnFamily ksAttrs = entry.getValue(); - - // we don't care about nested ColumnFamilies here because those are going to be processed separately - if (ksAttrs.hasColumns()) - addKeyspace(KSMetaData.fromSchema(new Row(entry.getKey(), entry.getValue()), Collections.<CFMetaData>emptyList(), new UTMetaData())); - } - - /** - * At second step we check if there were any keyspaces re-created, in this context - * re-created means that they were previously deleted but still exist in the low-level schema as empty keys - */ - - Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries = diff.entriesDiffering(); - - // instead of looping over all modified entries and skipping processed keys all the time - // we would rather store "left to process" items and iterate over them removing already met keys - List<DecoratedKey> leftToProcess = new ArrayList<DecoratedKey>(modifiedEntries.size()); - - for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> entry : modifiedEntries.entrySet()) - { - ColumnFamily prevValue = entry.getValue().leftValue(); - ColumnFamily newValue = entry.getValue().rightValue(); - - if (!prevValue.hasColumns()) - { - addKeyspace(KSMetaData.fromSchema(new Row(entry.getKey(), newValue), Collections.<CFMetaData>emptyList(), new UTMetaData())); - continue; - } - - leftToProcess.add(entry.getKey()); - } - - if (leftToProcess.size() == 0) - return Collections.emptySet(); - - /** - * At final step we updating modified keyspaces and saving keyspaces drop them later + List<Row> created = new ArrayList<>(); + List<String> altered = new ArrayList<>(); + Set<String> dropped = new HashSet<>(); + + /* + * - we don't care about entriesOnlyOnLeft() or entriesInCommon(), because only the changes are of interest to us + * - of all entriesOnlyOnRight(), we only care about ones that have live columns; it's possible to have a ColumnFamily + * there that only has the top-level deletion, if: + * a) a pushed DROP KEYSPACE change for a keyspace hadn't ever made it to this node in the first place + * b) a pulled dropped keyspace that got dropped before it could find a way to this node + * - of entriesDiffering(), we don't care about the scenario where both pre and post-values have zero live columns: + * that means that a keyspace had been recreated and dropped, and the recreated keyspace had never found a way + * to this node */ + MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(before, after); - Set<String> keyspacesToDrop = new HashSet<String>(); + for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().getColumnCount() > 0) ++ if (entry.getValue().hasColumns()) + created.add(new Row(entry.getKey(), entry.getValue())); - for (DecoratedKey key : leftToProcess) + for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> entry : diff.entriesDiffering().entrySet()) { - MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(key); - String keyspaceName = AsciiType.instance.compose(entry.getKey().key); ++ String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); - ColumnFamily newState = valueDiff.rightValue(); + ColumnFamily pre = entry.getValue().leftValue(); + ColumnFamily post = entry.getValue().rightValue(); - if (newState.hasColumns()) - updateKeyspace(KSMetaData.fromSchema(new Row(key, newState), Collections.<CFMetaData>emptyList(), new UTMetaData())); - else - keyspacesToDrop.add(AsciiType.instance.getString(key.getKey())); - if (pre.getColumnCount() > 0 && post.getColumnCount() > 0) ++ if (pre.hasColumns() && post.hasColumns()) + altered.add(keyspaceName); - else if (pre.getColumnCount() > 0) ++ else if (pre.hasColumns()) + dropped.add(keyspaceName); - else if (post.getColumnCount() > 0) // a (re)created keyspace ++ else if (post.hasColumns()) // a (re)created keyspace + created.add(new Row(entry.getKey(), post)); } - return keyspacesToDrop; + for (Row row : created) - addKeyspace(KSMetaData.fromSchema(row, Collections.<CFMetaData>emptyList())); ++ addKeyspace(KSMetaData.fromSchema(row, Collections.<CFMetaData>emptyList(), new UTMetaData())); + for (String name : altered) + updateKeyspace(name); + return dropped; } - private static void mergeColumnFamilies(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated) + // see the comments for mergeKeyspaces() + private static void mergeColumnFamilies(Map<DecoratedKey, ColumnFamily> before, Map<DecoratedKey, ColumnFamily> after) { - // calculate the difference between old and new states (note that entriesOnlyLeft() will be always empty) - MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated); + List<CFMetaData> created = new ArrayList<>(); + List<CFMetaData> altered = new ArrayList<>(); + List<CFMetaData> dropped = new ArrayList<>(); - // check if any new Keyspaces with ColumnFamilies were added. - for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet()) - { - ColumnFamily cfAttrs = entry.getValue(); + MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(before, after); - if (cfAttrs.hasColumns()) - { - Map<String, CFMetaData> cfDefs = KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(), cfAttrs)); - - for (CFMetaData cfDef : cfDefs.values()) - addColumnFamily(cfDef); - } - } - - // deal with modified ColumnFamilies (remember that all of the keyspace nested ColumnFamilies are put to the single row) - Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries = diff.entriesDiffering(); + for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet()) - if (entry.getValue().getColumnCount() > 0) ++ if (entry.getValue().hasColumns()) + created.addAll(KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(), entry.getValue())).values()); - for (DecoratedKey keyspace : modifiedEntries.keySet()) + for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> entry : diff.entriesDiffering().entrySet()) { - MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(keyspace); - - ColumnFamily prevValue = valueDiff.leftValue(); // state before external modification - ColumnFamily newValue = valueDiff.rightValue(); // updated state - String keyspaceName = AsciiType.instance.compose(entry.getKey().key); ++ String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); - Row newRow = new Row(keyspace, newValue); + ColumnFamily pre = entry.getValue().leftValue(); + ColumnFamily post = entry.getValue().rightValue(); - if (!prevValue.hasColumns()) // whole keyspace was deleted and now it's re-created - if (pre.getColumnCount() > 0 && post.getColumnCount() > 0) ++ if (pre.hasColumns() && post.hasColumns()) { - for (CFMetaData cfm : KSMetaData.deserializeColumnFamilies(newRow).values()) - addColumnFamily(cfm); + MapDifference<String, CFMetaData> delta = - Maps.difference(Schema.instance.getKSMetaData(keyspaceName).cfMetaData(), - KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(), post))); ++ Maps.difference(Schema.instance.getKSMetaData(keyspaceName).cfMetaData(), ++ KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(), post))); + + dropped.addAll(delta.entriesOnlyOnLeft().values()); + created.addAll(delta.entriesOnlyOnRight().values()); + Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function<MapDifference.ValueDifference<CFMetaData>, CFMetaData>() + { + public CFMetaData apply(MapDifference.ValueDifference<CFMetaData> pair) + { + return pair.rightValue(); + } + })); } - else if (!newValue.hasColumns()) // whole keyspace is deleted - else if (pre.getColumnCount() > 0) ++ else if (pre.hasColumns()) { - for (CFMetaData cfm : KSMetaData.deserializeColumnFamilies(new Row(keyspace, prevValue)).values()) - dropColumnFamily(cfm.ksName, cfm.cfName); + dropped.addAll(Schema.instance.getKSMetaData(keyspaceName).cfMetaData().values()); } - else // has modifications in the nested ColumnFamilies, need to perform nested diff to determine what was really changed - else if (post.getColumnCount() > 0) ++ else if (post.hasColumns()) { - String ksName = AsciiType.instance.getString(keyspace.getKey()); - - Map<String, CFMetaData> oldCfDefs = new HashMap<String, CFMetaData>(); - for (CFMetaData cfm : Schema.instance.getKSMetaData(ksName).cfMetaData().values()) - oldCfDefs.put(cfm.cfName, cfm); - - Map<String, CFMetaData> newCfDefs = KSMetaData.deserializeColumnFamilies(newRow); - - MapDifference<String, CFMetaData> cfDefDiff = Maps.difference(oldCfDefs, newCfDefs); - - for (CFMetaData cfDef : cfDefDiff.entriesOnlyOnRight().values()) - addColumnFamily(cfDef); - - for (CFMetaData cfDef : cfDefDiff.entriesOnlyOnLeft().values()) - dropColumnFamily(cfDef.ksName, cfDef.cfName); - - for (MapDifference.ValueDifference<CFMetaData> cfDef : cfDefDiff.entriesDiffering().values()) - updateColumnFamily(cfDef.rightValue()); + created.addAll(KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(), post)).values()); } } + + for (CFMetaData cfm : created) + addColumnFamily(cfm); + for (CFMetaData cfm : altered) + updateColumnFamily(cfm.ksName, cfm.cfName); + for (CFMetaData cfm : dropped) + dropColumnFamily(cfm.ksName, cfm.cfName); } - private static void mergeTypes(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated) ++ // see the comments for mergeKeyspaces() ++ private static void mergeTypes(Map<DecoratedKey, ColumnFamily> before, Map<DecoratedKey, ColumnFamily> after) + { - MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated); ++ List<UserType> created = new ArrayList<>(); ++ List<UserType> altered = new ArrayList<>(); ++ List<UserType> dropped = new ArrayList<>(); ++ ++ MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(before, after); + + // New keyspace with types + for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet()) - { - ColumnFamily cfTypes = entry.getValue(); - if (!cfTypes.hasColumns()) - continue; - - for (UserType ut : UTMetaData.fromSchema(new Row(entry.getKey(), cfTypes)).values()) - addType(ut); - } ++ if (entry.getValue().hasColumns()) ++ created.addAll(UTMetaData.fromSchema(new Row(entry.getKey(), entry.getValue())).values()); + - for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntry : diff.entriesDiffering().entrySet()) ++ for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> entry : diff.entriesDiffering().entrySet()) + { - DecoratedKey keyspace = modifiedEntry.getKey(); - ColumnFamily prevCFTypes = modifiedEntry.getValue().leftValue(); // state before external modification - ColumnFamily newCFTypes = modifiedEntry.getValue().rightValue(); // updated state ++ String keyspaceName = AsciiType.instance.compose(entry.getKey().getKey()); + - if (!prevCFTypes.hasColumns()) // whole keyspace was deleted and now it's re-created ++ ColumnFamily pre = entry.getValue().leftValue(); ++ ColumnFamily post = entry.getValue().rightValue(); ++ ++ if (pre.hasColumns() && post.hasColumns()) + { - for (UserType ut : UTMetaData.fromSchema(new Row(keyspace, newCFTypes)).values()) - addType(ut); ++ MapDifference<ByteBuffer, UserType> delta = ++ Maps.difference(Schema.instance.getKSMetaData(keyspaceName).userTypes.getAllTypes(), ++ UTMetaData.fromSchema(new Row(entry.getKey(), post))); ++ ++ dropped.addAll(delta.entriesOnlyOnLeft().values()); ++ created.addAll(delta.entriesOnlyOnRight().values()); ++ Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(), new Function<MapDifference.ValueDifference<UserType>, UserType>() ++ { ++ public UserType apply(MapDifference.ValueDifference<UserType> pair) ++ { ++ return pair.rightValue(); ++ } ++ })); + } - else if (!newCFTypes.hasColumns()) // whole keyspace is deleted ++ else if (pre.hasColumns()) + { - for (UserType ut : UTMetaData.fromSchema(new Row(keyspace, prevCFTypes)).values()) - dropType(ut); ++ dropped.addAll(Schema.instance.getKSMetaData(keyspaceName).userTypes.getAllTypes().values()); + } - else // has modifications in the types, need to perform nested diff to determine what was really changed ++ else if (post.hasColumns()) + { - MapDifference<ByteBuffer, UserType> typesDiff = Maps.difference(UTMetaData.fromSchema(new Row(keyspace, prevCFTypes)), - UTMetaData.fromSchema(new Row(keyspace, newCFTypes))); - - for (UserType type : typesDiff.entriesOnlyOnRight().values()) - addType(type); - - for (UserType type : typesDiff.entriesOnlyOnLeft().values()) - dropType(type); - - for (MapDifference.ValueDifference<UserType> tdiff : typesDiff.entriesDiffering().values()) - updateType(tdiff.rightValue()); // use the most recent value ++ created.addAll(UTMetaData.fromSchema(new Row(entry.getKey(), post)).values()); + } + } ++ ++ for (UserType type : created) ++ addType(type); ++ for (UserType type : altered) ++ updateType(type); ++ for (UserType type : dropped) ++ dropType(type); + } + private static void addKeyspace(KSMetaData ksm) { assert Schema.instance.getKSMetaData(ksm.name) == null; @@@ -412,22 -323,9 +387,22 @@@ } } + private static void addType(UserType ut) + { + KSMetaData ksm = Schema.instance.getKSMetaData(ut.keyspace); + assert ksm != null; + + logger.info("Loading {}", ut); + + ksm.userTypes.addType(ut); + + if (!StorageService.instance.isClientMode()) + MigrationManager.instance.notifyCreateUserType(ut); + } + - private static void updateKeyspace(KSMetaData newState) + private static void updateKeyspace(String ksName) { - KSMetaData oldKsm = Schema.instance.getKSMetaData(newState.name); + KSMetaData oldKsm = Schema.instance.getKSMetaData(ksName); assert oldKsm != null; KSMetaData newKsm = KSMetaData.cloneWith(oldKsm.reloadAttributes(), oldKsm.cfMetaData().values()); @@@ -549,7 -410,7 +524,7 @@@ private static KSMetaData makeNewKeyspaceDefinition(KSMetaData ksm, CFMetaData toExclude) { // clone ksm but do not include the new def -- List<CFMetaData> newCfs = new ArrayList<CFMetaData>(ksm.cfMetaData().values()); ++ List<CFMetaData> newCfs = new ArrayList<>(ksm.cfMetaData().values()); newCfs.remove(toExclude); assert newCfs.size() == ksm.cfMetaData().size() - 1; return KSMetaData.cloneWith(ksm, newCfs);