Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/47a2839b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/47a2839b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/47a2839b Branch: refs/heads/cassandra-3.11 Commit: 47a2839bf094a7fa9bad6de140fba486756c49bf Parents: 15abe2d 3960260 Author: AndreÌs de la PenÌa <a.penya.gar...@gmail.com> Authored: Tue Aug 8 15:01:04 2017 +0100 Committer: AndreÌs de la PenÌa <a.penya.gar...@gmail.com> Committed: Tue Aug 8 15:01:04 2017 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/config/ColumnDefinition.java | 19 ++++++++--- .../apache/cassandra/db/rows/AbstractCell.java | 2 +- .../org/apache/cassandra/db/rows/BTreeRow.java | 2 +- .../apache/cassandra/tools/JsonTransformer.java | 31 ++++++++++++++++-- .../org/apache/cassandra/cql3/ViewTest.java | 33 ++++++++++++++++++++ .../cassandra/index/sasi/SASIIndexTest.java | 14 +++++++++ 7 files changed, 94 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index b8c4bde,1525289..b778df6 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,9 -1,5 +1,10 @@@ -3.0.15 +3.11.1 + * "ignore" option is ignored in sstableloader (CASSANDRA-13721) + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652) + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512) + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641) +Merged from 3.0: + * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573) * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737) * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730) * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371) http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/config/ColumnDefinition.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/config/ColumnDefinition.java index ea508d2,6a0f530..159ea0c --- a/src/java/org/apache/cassandra/config/ColumnDefinition.java +++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java @@@ -446,197 -398,17 +448,206 @@@ public class ColumnDefinition extends C */ public AbstractType<?> cellValueType() { - return type instanceof CollectionType - ? ((CollectionType)type).valueComparator() - : type; + assert !(type instanceof UserType && type.isMultiCell()); + return type instanceof CollectionType && type.isMultiCell() + ? ((CollectionType)type).valueComparator() + : type; + } + + + public boolean isCounterColumn() + { + if (type instanceof CollectionType) // for thrift + return ((CollectionType) type).valueComparator().isCounter(); + return type.isCounter(); } + + public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames) throws InvalidRequestException + { + return SimpleSelector.newFactory(this, addAndGetIndex(this, defs)); + } + + public AbstractType<?> getExactTypeIfKnown(String keyspace) + { + return type; + } + + /** + * Because Thrift-created tables may have a non-text comparator, we cannot determine the proper 'key' until + * we know the comparator. ColumnDefinition.Raw is a placeholder that can be converted to a real ColumnIdentifier + * once the comparator is known with prepare(). This should only be used with identifiers that are actual + * column names. See CASSANDRA-8178 for more background. + */ + public static abstract class Raw extends Selectable.Raw + { + /** + * Creates a {@code ColumnDefinition.Raw} from an unquoted identifier string. + */ + public static Raw forUnquoted(String text) + { + return new Literal(text, false); + } + + /** + * Creates a {@code ColumnDefinition.Raw} from a quoted identifier string. + */ + public static Raw forQuoted(String text) + { + return new Literal(text, true); + } + + /** + * Creates a {@code ColumnDefinition.Raw} from a pre-existing {@code ColumnDefinition} + * (useful in the rare cases where we already have the column but need + * a {@code ColumnDefinition.Raw} for typing purposes). + */ + public static Raw forColumn(ColumnDefinition column) + { + return new ForColumn(column); + } + + /** + * Get the identifier corresponding to this raw column, without assuming this is an + * existing column (unlike {@link #prepare}). + */ + public abstract ColumnIdentifier getIdentifier(CFMetaData cfm); + + public abstract String rawText(); + + @Override + public abstract ColumnDefinition prepare(CFMetaData cfm); + + @Override + public boolean processesSelection() + { + return false; + } + + @Override + public final int hashCode() + { + return toString().hashCode(); + } + + @Override + public final boolean equals(Object o) + { + if(!(o instanceof Raw)) + return false; + + Raw that = (Raw)o; + return this.toString().equals(that.toString()); + } + + private static class Literal extends Raw + { + private final String text; + + public Literal(String rawText, boolean keepCase) + { + this.text = keepCase ? rawText : rawText.toLowerCase(Locale.US); + } + + public ColumnIdentifier getIdentifier(CFMetaData cfm) + { + if (!cfm.isStaticCompactTable()) + return ColumnIdentifier.getInterned(text, true); + + AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType(); + if (thriftColumnNameType instanceof UTF8Type) + return ColumnIdentifier.getInterned(text, true); + + // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use + // thriftColumnNameType + ByteBuffer bufferName = ByteBufferUtil.bytes(text); + for (ColumnDefinition def : cfm.allColumns()) + { + if (def.name.bytes.equals(bufferName)) + return def.name; + } + return ColumnIdentifier.getInterned(thriftColumnNameType, thriftColumnNameType.fromString(text), text); + } + + public ColumnDefinition prepare(CFMetaData cfm) + { + if (!cfm.isStaticCompactTable()) + return find(cfm); + + AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType(); + if (thriftColumnNameType instanceof UTF8Type) + return find(cfm); + + // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use + // thriftColumnNameType + ByteBuffer bufferName = ByteBufferUtil.bytes(text); + for (ColumnDefinition def : cfm.allColumns()) + { + if (def.name.bytes.equals(bufferName)) + return def; + } + return find(thriftColumnNameType.fromString(text), cfm); + } + + private ColumnDefinition find(CFMetaData cfm) + { + return find(ByteBufferUtil.bytes(text), cfm); + } + + private ColumnDefinition find(ByteBuffer id, CFMetaData cfm) + { + ColumnDefinition def = cfm.getColumnDefinition(id); + if (def == null) + throw new InvalidRequestException(String.format("Undefined column name %s", toString())); + return def; + } + + public String rawText() + { + return text; + } + + @Override + public String toString() + { + return ColumnIdentifier.maybeQuote(text); + } + } + + // Use internally in the rare case where we need a ColumnDefinition.Raw for type-checking but + // actually already have the column itself. + private static class ForColumn extends Raw + { + private final ColumnDefinition column; + + private ForColumn(ColumnDefinition column) + { + this.column = column; + } + + public ColumnIdentifier getIdentifier(CFMetaData cfm) + { + return column.name; + } + + public ColumnDefinition prepare(CFMetaData cfm) + { + assert cfm.getColumnDefinition(column.name) != null; // Sanity check that we're not doing something crazy + return column; + } + + public String rawText() + { + return column.name.toString(); + } + + @Override + public String toString() + { + return column.name.toCQLString(); + } + } + } + + + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/AbstractCell.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/rows/AbstractCell.java index 54bd9e8,7e93c2e..54c8f24 --- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java +++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java @@@ -44,81 -40,6 +44,81 @@@ public abstract class AbstractCell exte super(column); } + public boolean isCounterCell() + { - return !isTombstone() && column.cellValueType().isCounter(); ++ return !isTombstone() && column.isCounterColumn(); + } + + public boolean isLive(int nowInSec) + { + return localDeletionTime() == NO_DELETION_TIME || (ttl() != NO_TTL && nowInSec < localDeletionTime()); + } + + public boolean isTombstone() + { + return localDeletionTime() != NO_DELETION_TIME && ttl() == NO_TTL; + } + + public boolean isExpiring() + { + return ttl() != NO_TTL; + } + + public Cell markCounterLocalToBeCleared() + { + if (!isCounterCell()) + return this; + + ByteBuffer value = value(); + ByteBuffer marked = CounterContext.instance().markLocalToBeCleared(value); + return marked == value ? this : new BufferCell(column, timestamp(), ttl(), localDeletionTime(), marked, path()); + } + + public Cell purge(DeletionPurger purger, int nowInSec) + { + if (!isLive(nowInSec)) + { + if (purger.shouldPurge(timestamp(), localDeletionTime())) + return null; + + // We slightly hijack purging to convert expired but not purgeable columns to tombstones. The reason we do that is + // that once a column has expired it is equivalent to a tombstone but actually using a tombstone is more compact since + // we don't keep the column value. The reason we do it here is that 1) it's somewhat related to dealing with tombstones + // so hopefully not too surprising and 2) we want to this and purging at the same places, so it's simpler/more efficient + // to do both here. + if (isExpiring()) + { + // Note that as long as the expiring column and the tombstone put together live longer than GC grace seconds, + // we'll fulfil our responsibility to repair. See discussion at + // http://cassandra-user-incubator-apache-org.3065146.n2.nabble.com/repair-compaction-and-tombstone-rows-td7583481.html + return BufferCell.tombstone(column, timestamp(), localDeletionTime() - ttl(), path()).purge(purger, nowInSec); + } + } + return this; + } + + public Cell copy(AbstractAllocator allocator) + { + CellPath path = path(); + return new BufferCell(column, timestamp(), ttl(), localDeletionTime(), allocator.clone(value()), path == null ? null : path.copy(allocator)); + } + + // note: while the cell returned may be different, the value is the same, so if the value is offheap it must be referenced inside a guarded context (or copied) + public Cell updateAllTimestamp(long newTimestamp) + { + return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime(), value(), path()); + } + + public int dataSize() + { + CellPath path = path(); + return TypeSizes.sizeof(timestamp()) + + TypeSizes.sizeof(ttl()) + + TypeSizes.sizeof(localDeletionTime()) + + value().remaining() + + (path == null ? 0 : path.dataSize()); + } + public void digest(MessageDigest digest) { digest.update(value().duplicate()); http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/BTreeRow.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/tools/JsonTransformer.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/tools/JsonTransformer.java index 360e8ff,5c32035..c3a0a17 --- a/src/java/org/apache/cassandra/tools/JsonTransformer.java +++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java @@@ -46,6 -50,7 +47,7 @@@ import org.apache.cassandra.db.rows.Row import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.transport.Server; ++import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@@ -405,9 -411,9 +407,10 @@@ public final class JsonTransforme objectIndenter.setCompact(true); json.writeFieldName("name"); AbstractType<?> type = cell.column().type; ++ AbstractType<?> cellType = null; json.writeString(cell.column().name.toCQLString()); - if (cell.path() != null && cell.path().size() > 0) + if (type.isCollection() && type.isMultiCell()) // non-frozen collection { CollectionType ct = (CollectionType) type; json.writeFieldName("path"); @@@ -419,6 -425,6 +422,30 @@@ } json.writeEndArray(); arrayIndenter.setCompact(false); ++ ++ cellType = cell.column().cellValueType(); ++ } ++ else if (type.isUDT() && type.isMultiCell()) // non-frozen udt ++ { ++ UserType ut = (UserType) type; ++ json.writeFieldName("path"); ++ arrayIndenter.setCompact(true); ++ json.writeStartArray(); ++ for (int i = 0; i < cell.path().size(); i++) ++ { ++ Short fieldPosition = ut.nameComparator().compose(cell.path().get(i)); ++ json.writeString(ut.fieldNameAsString(fieldPosition)); ++ } ++ json.writeEndArray(); ++ arrayIndenter.setCompact(false); ++ ++ // cellType of udt ++ Short fieldPosition = ((UserType) type).nameComparator().compose(cell.path().get(0)); ++ cellType = ((UserType) type).fieldType(fieldPosition); ++ } ++ else ++ { ++ cellType = cell.column().cellValueType(); } if (cell.isTombstone()) { @@@ -433,7 -439,7 +460,7 @@@ else { json.writeFieldName("value"); - json.writeString(cell.column().cellValueType().getString(cell.value())); - json.writeRawValue(cell.column().cellValueType().toJSONString(cell.value(), Server.CURRENT_VERSION)); ++ json.writeRawValue(cellType.toJSONString(cell.value(), ProtocolVersion.CURRENT)); } if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp()) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/test/unit/org/apache/cassandra/cql3/ViewTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/ViewTest.java index 6f6e04d,f8f8c9f..0853562 --- a/test/unit/org/apache/cassandra/cql3/ViewTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java @@@ -777,6 -788,6 +777,39 @@@ public class ViewTest extends CQLTeste } @Test ++ public void testFrozenCollectionsWithComplicatedInnerType() throws Throwable ++ { ++ createTable("CREATE TABLE %s (k int, intval int, listval frozen<list<tuple<text,text>>>, PRIMARY KEY (k))"); ++ ++ execute("USE " + keyspace()); ++ executeNet(protocolVersion, "USE " + keyspace()); ++ ++ createView("mv", ++ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND listval IS NOT NULL PRIMARY KEY (k, listval)"); ++ ++ updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))", ++ 0, ++ 0, ++ "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]"); ++ ++ // verify input ++ assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), ++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3")))); ++ assertRows(execute("SELECT k, listval from mv"), ++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3")))); ++ ++ // update listval with the same value and it will be compared in view generator ++ updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", ++ 0, ++ "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]"); ++ // verify result ++ assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), ++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3")))); ++ assertRows(execute("SELECT k, listval from mv"), ++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3")))); ++ } ++ ++ @Test public void testUpdate() throws Throwable { createTable("CREATE TABLE %s (" + --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org