http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java index d0e66f8,0000000..e0cbc0f mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java @@@ -1,245 -1,0 +1,301 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.composites; + +import java.nio.ByteBuffer; +import java.util.*; + ++import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.CQL3Row; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.ColumnToCollectionType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.utils.ByteBufferUtil; ++import org.apache.cassandra.utils.memory.AbstractAllocator; ++import org.apache.cassandra.utils.memory.PoolAllocator; + +public class CompoundSparseCellNameType extends AbstractCompoundCellNameType +{ + private static final ColumnIdentifier rowMarkerId = new ColumnIdentifier(ByteBufferUtil.EMPTY_BYTE_BUFFER, UTF8Type.instance); - private static final CellName rowMarkerNoPrefix = new CompoundSparseCellName(rowMarkerId); ++ private static final CellName rowMarkerNoPrefix = new CompoundSparseCellName(rowMarkerId, false); + + // For CQL3 columns, this is always UTF8Type. However, for compatibility with super columns, we need to allow it to be non-UTF8. + private final AbstractType<?> columnNameType; + protected final Map<ByteBuffer, ColumnIdentifier> internedIds; + ++ private final Composite staticPrefix; ++ + public CompoundSparseCellNameType(List<AbstractType<?>> types) + { + this(types, UTF8Type.instance); + } + + public CompoundSparseCellNameType(List<AbstractType<?>> types, AbstractType<?> columnNameType) + { + this(new CompoundCType(types), columnNameType); + } + + private CompoundSparseCellNameType(CompoundCType clusteringType, AbstractType<?> columnNameType) + { + this(clusteringType, columnNameType, makeCType(clusteringType, columnNameType, null), new HashMap<ByteBuffer, ColumnIdentifier>()); + } + + private CompoundSparseCellNameType(CompoundCType clusteringType, AbstractType<?> columnNameType, CompoundCType fullType, Map<ByteBuffer, ColumnIdentifier> internedIds) + { + super(clusteringType, fullType); + this.columnNameType = columnNameType; + this.internedIds = internedIds; ++ this.staticPrefix = makeStaticPrefix(clusteringType.size()); ++ } ++ ++ private static Composite makeStaticPrefix(int size) ++ { ++ ByteBuffer[] elements = new ByteBuffer[size]; ++ for (int i = 0; i < size; i++) ++ elements[i] = ByteBufferUtil.EMPTY_BYTE_BUFFER; ++ ++ return new CompoundComposite(elements, size, true) ++ { ++ @Override ++ public boolean isStatic() ++ { ++ return true; ++ } ++ ++ @Override ++ public long unsharedHeapSize() ++ { ++ // We'll share this for a given type. ++ return 0; ++ } ++ ++ @Override ++ public Composite copy(AbstractAllocator allocator) ++ { ++ return this; ++ } ++ ++ @Override ++ public void free(PoolAllocator<?> allocator) ++ { ++ } ++ }; + } + + protected static CompoundCType makeCType(CompoundCType clusteringType, AbstractType<?> columnNameType, ColumnToCollectionType collectionType) + { + List<AbstractType<?>> allSubtypes = new ArrayList<AbstractType<?>>(clusteringType.size() + (collectionType == null ? 1 : 2)); + for (int i = 0; i < clusteringType.size(); i++) + allSubtypes.add(clusteringType.subtype(i)); + allSubtypes.add(columnNameType); + if (collectionType != null) + allSubtypes.add(collectionType); + return new CompoundCType(allSubtypes); + } + + public CellNameType setSubtype(int position, AbstractType<?> newType) + { + if (position < clusteringSize) + return new CompoundSparseCellNameType(clusteringType.setSubtype(position, newType), columnNameType, fullType.setSubtype(position, newType), internedIds); + + if (position == clusteringSize) + throw new IllegalArgumentException(); + + throw new IndexOutOfBoundsException(); + } + + @Override + public CellNameType addCollection(ColumnIdentifier columnName, CollectionType newCollection) + { + return new WithCollection(clusteringType, ColumnToCollectionType.getInstance(Collections.singletonMap(columnName.bytes, newCollection)), internedIds); + } + + public boolean isDense() + { + return false; + } + + public boolean supportCollections() + { + return true; + } + - public CellName create(Composite prefix, ColumnIdentifier columnName) ++ public Composite staticPrefix() ++ { ++ return staticPrefix; ++ } ++ ++ public CellName create(Composite prefix, ColumnDefinition column) + { ++ return create(prefix, column.name, column.isStatic()); ++ } ++ ++ private CellName create(Composite prefix, ColumnIdentifier columnName, boolean isStatic) ++ { ++ if (isStatic) ++ prefix = staticPrefix(); ++ + assert prefix.size() == clusteringSize; + + if (prefix.isEmpty()) - return new CompoundSparseCellName(columnName); ++ return new CompoundSparseCellName(columnName, isStatic); + + assert prefix instanceof CompoundComposite; + CompoundComposite lc = (CompoundComposite)prefix; - return new CompoundSparseCellName(lc.elements, clusteringSize, columnName); ++ return new CompoundSparseCellName(lc.elements, clusteringSize, columnName, isStatic); + } + + public CellName rowMarker(Composite prefix) + { ++ assert !prefix.isStatic(); // static columns don't really create rows, they shouldn't have a row marker + if (prefix.isEmpty()) + return rowMarkerNoPrefix; + - return create(prefix, rowMarkerId); ++ return create(prefix, rowMarkerId, false); + } + + protected ColumnIdentifier idFor(ByteBuffer bb) + { + ColumnIdentifier id = internedIds.get(bb); + return id == null ? new ColumnIdentifier(bb, columnNameType) : id; + } + - protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc) ++ protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) + { + if (size < clusteringSize + 1 || eoc != Composite.EOC.NONE) - return new CompoundComposite(components, size).withEOC(eoc); ++ return new CompoundComposite(components, size, isStatic).withEOC(eoc); + - return new CompoundSparseCellName(components, clusteringSize, idFor(components[clusteringSize])); ++ return new CompoundSparseCellName(components, clusteringSize, idFor(components[clusteringSize]), isStatic); + } + - protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc) ++ protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) + { + if (size < clusteringSize + 1 || eoc != Composite.EOC.NONE) - return new CompoundComposite(Arrays.copyOfRange(components, 0, size), size).withEOC(eoc); ++ return new CompoundComposite(Arrays.copyOfRange(components, 0, size), size, isStatic).withEOC(eoc); + + ByteBuffer[] clusteringColumns = Arrays.copyOfRange(components, 0, clusteringSize); - return new CompoundSparseCellName(clusteringColumns, idFor(components[clusteringSize])); ++ return new CompoundSparseCellName(clusteringColumns, idFor(components[clusteringSize]), isStatic); + } + + public void addCQL3Column(ColumnIdentifier id) + { + internedIds.put(id.bytes, id); + } + + public void removeCQL3Column(ColumnIdentifier id) + { + internedIds.remove(id.bytes); + } + + public CQL3Row.Builder CQL3RowBuilder(long now) + { + return makeSparseCQL3RowBuilder(now); + } + + public static class WithCollection extends CompoundSparseCellNameType + { + private final ColumnToCollectionType collectionType; + + public WithCollection(List<AbstractType<?>> types, ColumnToCollectionType collectionType) + { + this(new CompoundCType(types), collectionType); + } + + WithCollection(CompoundCType clusteringType, ColumnToCollectionType collectionType) + { + this(clusteringType, collectionType, new HashMap<ByteBuffer, ColumnIdentifier>()); + } + + private WithCollection(CompoundCType clusteringType, ColumnToCollectionType collectionType, Map<ByteBuffer, ColumnIdentifier> internedIds) + { + this(clusteringType, makeCType(clusteringType, UTF8Type.instance, collectionType), collectionType, internedIds); + } + + private WithCollection(CompoundCType clusteringType, CompoundCType fullCType, ColumnToCollectionType collectionType, Map<ByteBuffer, ColumnIdentifier> internedIds) + { + super(clusteringType, UTF8Type.instance, fullCType, internedIds); + this.collectionType = collectionType; + } + + @Override + public CellNameType setSubtype(int position, AbstractType<?> newType) + { + if (position < clusteringSize) + return new WithCollection(clusteringType.setSubtype(position, newType), collectionType, internedIds); + + throw position >= fullType.size() ? new IndexOutOfBoundsException() : new IllegalArgumentException(); + } + + @Override + public CellNameType addCollection(ColumnIdentifier columnName, CollectionType newCollection) + { + Map<ByteBuffer, CollectionType> newMap = new HashMap<>(collectionType.defined); + newMap.put(columnName.bytes, newCollection); + return new WithCollection(clusteringType, ColumnToCollectionType.getInstance(newMap), internedIds); + } + + @Override - public CellName create(Composite prefix, ColumnIdentifier columnName, ByteBuffer collectionElement) ++ public CellName create(Composite prefix, ColumnDefinition column, ByteBuffer collectionElement) + { - // We ignore the columnName because it's just the COMPACT_VALUE name which is not store in the cell name ++ if (column.isStatic()) ++ prefix = staticPrefix(); ++ + assert prefix.size() == clusteringSize; + + if (prefix.isEmpty()) - return new CompoundSparseCellName.WithCollection(columnName, collectionElement); ++ return new CompoundSparseCellName.WithCollection(column.name, collectionElement, column.isStatic()); + + assert prefix instanceof CompoundComposite; + CompoundComposite lc = (CompoundComposite)prefix; - return new CompoundSparseCellName.WithCollection(lc.elements, clusteringSize, columnName, collectionElement); ++ return new CompoundSparseCellName.WithCollection(lc.elements, clusteringSize, column.name, collectionElement, column.isStatic()); + } + + @Override + public boolean hasCollections() + { + return true; + } + + @Override + public ColumnToCollectionType collectionType() + { + return collectionType; + } + + @Override - protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc) ++ protected Composite makeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) + { + if (size < fullSize) - return super.makeWith(components, size, eoc); ++ return super.makeWith(components, size, eoc, isStatic); + - return new CompoundSparseCellName.WithCollection(components, clusteringSize, idFor(components[clusteringSize]), components[fullSize - 1]); ++ return new CompoundSparseCellName.WithCollection(components, clusteringSize, idFor(components[clusteringSize]), components[fullSize - 1], isStatic); + } + - protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc) ++ protected Composite copyAndMakeWith(ByteBuffer[] components, int size, Composite.EOC eoc, boolean isStatic) + { + if (size < fullSize) - return super.copyAndMakeWith(components, size, eoc); ++ return super.copyAndMakeWith(components, size, eoc, isStatic); + + ByteBuffer[] clusteringColumns = Arrays.copyOfRange(components, 0, clusteringSize); - return new CompoundSparseCellName.WithCollection(clusteringColumns, idFor(components[clusteringSize]), components[clusteringSize + 1]); ++ return new CompoundSparseCellName.WithCollection(clusteringColumns, idFor(components[clusteringSize]), components[clusteringSize + 1], isStatic); + } + } +} +
http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java index cafb521,0000000..1aae580 mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleDenseCellNameType.java @@@ -1,78 -1,0 +1,79 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.composites; + +import java.nio.ByteBuffer; + ++import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.CQL3Row; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.marshal.AbstractType; + +public class SimpleDenseCellNameType extends AbstractSimpleCellNameType +{ + public SimpleDenseCellNameType(AbstractType<?> type) + { + super(type); + } + + public int clusteringPrefixSize() + { + return 1; + } + + public CBuilder prefixBuilder() + { + // Simple dense is "all" prefix + return builder(); + } + + public CellNameType setSubtype(int position, AbstractType<?> newType) + { + if (position != 0) + throw new IllegalArgumentException(); + return new SimpleDenseCellNameType(newType); + } + + public boolean isDense() + { + return true; + } + - public CellName create(Composite prefix, ColumnIdentifier columnName) ++ public CellName create(Composite prefix, ColumnDefinition column) + { + assert prefix.size() == 1; - // We ignore the columnName because it's just the COMPACT_VALUE name which is not store in the cell name ++ // We ignore the column because it's just the COMPACT_VALUE name which is not store in the cell name + return new SimpleDenseCellName(prefix.get(0)); + } + + @Override + public Composite fromByteBuffer(ByteBuffer bb) + { + return !bb.hasRemaining() + ? Composites.EMPTY + : new SimpleDenseCellName(bb); + } + + public void addCQL3Column(ColumnIdentifier id) {} + public void removeCQL3Column(ColumnIdentifier id) {} + + public CQL3Row.Builder CQL3RowBuilder(long now) + { + return makeDenseCQL3RowBuilder(now); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java index 9c99680,0000000..0f63a6a mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellNameType.java @@@ -1,98 -1,0 +1,99 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.composites; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + ++import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.cql3.CQL3Row; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.marshal.AbstractType; + +public class SimpleSparseCellNameType extends AbstractSimpleCellNameType +{ + // Simple sparse means static thrift CF or non-clustered CQL3. This means that cell names will mainly + // be those that have been declared and we can intern the whole CellName instances. + private final Map<ByteBuffer, CellName> internedNames; + + public SimpleSparseCellNameType(AbstractType<?> type) + { + this(type, new HashMap<ByteBuffer, CellName>()); + } + + private SimpleSparseCellNameType(AbstractType<?> type, Map<ByteBuffer, CellName> internedNames) + { + super(type); + this.internedNames = internedNames; + } + + public int clusteringPrefixSize() + { + return 0; + } + + public CellNameType setSubtype(int position, AbstractType<?> newType) + { + if (position != 0) + throw new IllegalArgumentException(); + return new SimpleSparseCellNameType(newType, internedNames); + } + + public CBuilder prefixBuilder() + { + return Composites.EMPTY_BUILDER; + } + + public boolean isDense() + { + return false; + } + - public CellName create(Composite prefix, ColumnIdentifier columnName) ++ public CellName create(Composite prefix, ColumnDefinition column) + { + assert prefix.isEmpty(); - CellName cn = internedNames.get(columnName.bytes); - return cn == null ? new SimpleSparseCellName(columnName) : cn; ++ CellName cn = internedNames.get(column.name.bytes); ++ return cn == null ? new SimpleSparseCellName(column.name) : cn; + } + + @Override + public Composite fromByteBuffer(ByteBuffer bb) + { + if (!bb.hasRemaining()) + return Composites.EMPTY; + + CellName cn = internedNames.get(bb); + return cn == null ? new SimpleSparseCellName(new ColumnIdentifier(bb, type)) : cn; + } + + public void addCQL3Column(ColumnIdentifier id) + { + internedNames.put(id.bytes, new SimpleSparseInternedCellName(id)); + } + + public void removeCQL3Column(ColumnIdentifier id) + { + internedNames.remove(id.bytes); + } + + public CQL3Row.Builder CQL3RowBuilder(long now) + { + return makeSparseCQL3RowBuilder(now); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/ExtendedFilter.java index 29976f6,5aa1ea9..7326d80 --- a/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ExtendedFilter.java @@@ -322,49 -320,20 +322,49 @@@ public abstract class ExtendedFilte return true; } - private ByteBuffer extractDataValue(ColumnDefinition def, ByteBuffer rowKey, ColumnFamily data, ColumnNameBuilder builder) + private static boolean collectionSatisfies(ColumnDefinition def, ColumnFamily data, Composite prefix, IndexExpression expr, ByteBuffer collectionElement) + { + assert def.type.isCollection(); + + CollectionType type = (CollectionType)def.type; + switch (type.kind) + { + case LIST: + assert collectionElement != null; - return type.valueComparator().compare(data.getColumn(data.getComparator().create(prefix, def.name, collectionElement)).value(), expr.value) == 0; ++ return type.valueComparator().compare(data.getColumn(data.getComparator().create(prefix, def, collectionElement)).value(), expr.value) == 0; + case SET: - return data.getColumn(data.getComparator().create(prefix, def.name, expr.value)) != null; ++ return data.getColumn(data.getComparator().create(prefix, def, expr.value)) != null; + case MAP: + if (expr.operator == IndexExpression.Operator.CONTAINS_KEY) + { - return data.getColumn(data.getComparator().create(prefix, def.name, expr.value)) != null; ++ return data.getColumn(data.getComparator().create(prefix, def, expr.value)) != null; + } + else + { + assert collectionElement != null; - return type.valueComparator().compare(data.getColumn(data.getComparator().create(prefix, def.name, collectionElement)).value(), expr.value) == 0; ++ return type.valueComparator().compare(data.getColumn(data.getComparator().create(prefix, def, collectionElement)).value(), expr.value) == 0; + } + } + throw new AssertionError(); + } + + private ByteBuffer extractDataValue(ColumnDefinition def, ByteBuffer rowKey, ColumnFamily data, Composite prefix) { - switch (def.type) + switch (def.kind) { case PARTITION_KEY: - return def.componentIndex == null + return def.isOnAllComponents() ? rowKey - : ((CompositeType)data.metadata().getKeyValidator()).split(rowKey)[def.componentIndex]; - case CLUSTERING_KEY: - return builder.get(def.componentIndex); + : ((CompositeType)data.metadata().getKeyValidator()).split(rowKey)[def.position()]; + case CLUSTERING_COLUMN: + return prefix.get(def.position()); case REGULAR: - ByteBuffer colName = builder == null ? def.name : builder.copy().add(def.name).build(); - Column column = data.getColumn(colName); - return column == null ? null : column.value(); + CellName cname = prefix == null + ? data.getComparator().cellFromByteBuffer(def.name.bytes) - : data.getComparator().create(prefix, def.name); ++ : data.getComparator().create(prefix, def); + + Cell cell = data.getColumn(cname); + return cell == null ? null : cell.value(); case COMPACT_VALUE: assert data.getColumnCount() == 1; return data.getSortedColumns().iterator().next().value(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java index 36504ca,0000000..c8fc56c mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionKey.java @@@ -1,106 -1,0 +1,106 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.index.composites; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.composites.CBuilder; +import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.composites.CompoundDenseCellNameType; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.marshal.*; + +/** + * Index on the collection element of the cell name of a collection. + * + * A cell indexed by this index will have the general form: + * ck_0 ... ck_n c_name [col_elt] : v + * where ck_i are the cluster keys, c_name the CQL3 column name, col_elt the + * collection element that we want to index (which may or may not be there depending + * on whether c_name is the collection we're indexing) and v the cell value. + * + * Such a cell is indexed if c_name is the indexed collection (in which case we are guaranteed to have + * col_elt). The index entry will be: + * - row key will be col_elt value (getIndexedValue()). + * - cell name will be 'rk ck_0 ... ck_n' where rk is the row key of the initial cell. + */ +public class CompositesIndexOnCollectionKey extends CompositesIndex +{ + public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + { + int count = 1 + baseMetadata.clusteringColumns().size(); // row key + clustering prefix + List<AbstractType<?>> types = new ArrayList<AbstractType<?>>(count); + types.add(SecondaryIndex.keyComparator); + for (int i = 0; i < count - 1; i++) + types.add(baseMetadata.comparator.subtype(i)); + return new CompoundDenseCellNameType(types); + } + + @Override + protected AbstractType<?> getIndexKeyComparator() + { + return ((CollectionType)columnDef.type).nameComparator(); + } + + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + { + return cell.name().get(columnDef.position() + 1); + } + + protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite cellName) + { + int count = 1 + baseCfs.metadata.clusteringColumns().size(); + CBuilder builder = getIndexComparator().builder(); + builder.add(rowKey); + for (int i = 0; i < count - 1; i++) + builder.add(cellName.get(i)); + return builder.build(); + } + + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + { + int count = 1 + baseCfs.metadata.clusteringColumns().size(); + CBuilder builder = baseCfs.getComparator().builder(); + for (int i = 0; i < count - 1; i++) + builder.add(indexEntry.name().get(i + 1)); + return new IndexedEntry(indexedValue, indexEntry.name(), indexEntry.timestamp(), indexEntry.name().get(0), builder.build()); + } + + @Override + public boolean indexes(CellName name) + { + // We index if the CQL3 column name is the one of the collection we index + AbstractType<?> comp = baseCfs.metadata.getColumnDefinitionComparator(columnDef); + return name.size() > columnDef.position() + && comp.compare(name.get(columnDef.position()), columnDef.name.bytes) == 0; + } + + public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef.name, entry.indexValue.key); ++ CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef, entry.indexValue.key); + Cell liveCell = data.getColumn(name); + return (liveCell == null || liveCell.isMarkedForDelete(now)); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java index 1d780cd,0000000..9536e2e mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnCollectionValue.java @@@ -1,108 -1,0 +1,108 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.index.composites; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.composites.CBuilder; +import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.composites.CompoundDenseCellNameType; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.marshal.*; + +/** + * Index the value of a collection cell. + * + * This is a lot like an index on REGULAR, except that we also need to make + * the collection key part of the index entry so that: + * 1) we don't have to scan the whole collection at query time to know the + * entry is stale and if it still satisfies the query. + * 2) if a collection has multiple time the same value, we need one entry + * for each so that if we delete one of the value only we only delete the + * entry corresponding to that value. + */ +public class CompositesIndexOnCollectionValue extends CompositesIndex +{ + public static CellNameType buildIndexComparator(CFMetaData baseMetadata, ColumnDefinition columnDef) + { + int prefixSize = columnDef.position(); + List<AbstractType<?>> types = new ArrayList<AbstractType<?>>(prefixSize + 2); + types.add(SecondaryIndex.keyComparator); + for (int i = 0; i < prefixSize; i++) + types.add(baseMetadata.comparator.subtype(i)); + types.add(((CollectionType)columnDef.type).nameComparator()); // collection key + return new CompoundDenseCellNameType(types); + } + + @Override + protected AbstractType<?> getIndexKeyComparator() + { + return ((CollectionType)columnDef.type).valueComparator(); + } + + protected ByteBuffer getIndexedValue(ByteBuffer rowKey, Cell cell) + { + return cell.value(); + } + + protected Composite makeIndexColumnPrefix(ByteBuffer rowKey, Composite cellName) + { + CBuilder builder = getIndexComparator().prefixBuilder(); + builder.add(rowKey); + for (int i = 0; i < Math.min(columnDef.position(), cellName.size()); i++) + builder.add(cellName.get(i)); + builder.add(cellName.get(columnDef.position() + 1)); + return builder.build(); + } + + public IndexedEntry decodeEntry(DecoratedKey indexedValue, Cell indexEntry) + { + int prefixSize = columnDef.position(); + CellName name = indexEntry.name(); + CBuilder builder = baseCfs.getComparator().builder(); + for (int i = 0; i < prefixSize; i++) + builder.add(name.get(i + 1)); + return new IndexedEntry(indexedValue, name, indexEntry.timestamp(), name.get(0), builder.build(), name.get(prefixSize + 1)); + } + + @Override + public boolean indexes(CellName name) + { + AbstractType<?> comp = baseCfs.metadata.getColumnDefinitionComparator(columnDef); + return name.size() > columnDef.position() + && comp.compare(name.get(columnDef.position()), columnDef.name.bytes) == 0; + } + + public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) + { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef.name, entry.indexedEntryCollectionKey); ++ CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef, entry.indexedEntryCollectionKey); + Cell liveCell = data.getColumn(name); + if (liveCell == null || liveCell.isMarkedForDelete(now)) + return true; + + ByteBuffer liveValue = liveCell.value(); + return ((CollectionType)columnDef.type).valueComparator().compare(entry.indexValue.key, liveValue) != 0; + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java index f1b0954,7159c23..fc2f9db --- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndexOnRegular.java @@@ -89,12 -93,12 +89,12 @@@ public class CompositesIndexOnRegular e public boolean isStale(IndexedEntry entry, ColumnFamily data, long now) { - CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef.name); - ByteBuffer bb = entry.indexedEntryNameBuilder.copy().add(columnDef.name).build(); - Column liveColumn = data.getColumn(bb); - if (liveColumn == null || liveColumn.isMarkedForDelete(now)) ++ CellName name = data.getComparator().create(entry.indexedEntryPrefix, columnDef); + Cell liveCell = data.getColumn(name); + if (liveCell == null || liveCell.isMarkedForDelete(now)) return true; - ByteBuffer liveValue = liveColumn.value(); - return columnDef.getValidator().compare(entry.indexValue.key, liveValue) != 0; + ByteBuffer liveValue = liveCell.value(); + return columnDef.type.compare(entry.indexValue.key, liveValue) != 0; } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java index 41c9f41,eb618f4..e09b2d1 --- a/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java +++ b/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java @@@ -24,17 -24,14 +24,14 @@@ import java.util.* import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.cql3.ColumnNameBuilder; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.composites.CellNameType; +import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.composites.Composites; - import org.apache.cassandra.db.filter.ExtendedFilter; - import org.apache.cassandra.db.filter.IDiskAtomFilter; - import org.apache.cassandra.db.filter.QueryFilter; - import org.apache.cassandra.db.filter.SliceQueryFilter; + import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.index.SecondaryIndexSearcher; -import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.thrift.IndexExpression; import org.apache.cassandra.utils.ByteBufferUtil; public class CompositesSearcher extends SecondaryIndexSearcher @@@ -249,11 -235,28 +246,21 @@@ // We always query the whole CQL3 row. In the case where the original filter was a name filter this might be // slightly wasteful, but this probably doesn't matter in practice and it simplify things. - SliceQueryFilter dataFilter = new SliceQueryFilter(start, - entry.indexedEntryPrefix.end(), - false, - Integer.MAX_VALUE, - baseCfs.metadata.clusteringColumns().size()); - ColumnSlice dataSlice = new ColumnSlice(start, entry.indexedEntryEnd()); - ColumnSlice[] slices; - if (baseCfs.metadata.hasStaticColumns()) - { - // If the table has static columns, we must fetch them too as they may need to be returned too. - // Note that this is potentially wasteful for 2 reasons: - // 1) we will retrieve the static parts for each indexed row, even if we have more than one row in - // the same partition. If we were to group data queries to rows on the same slice, which would - // speed up things in general, we would also optimize here since we would fetch static columns only - // once for each group. - // 2) at this point we don't know if the user asked for static columns or not, so we might be fetching - // them for nothing. We would however need to ship the list of "CQL3 columns selected" with getRangeSlice - // to be able to know that. - // TODO: we should improve both point above - ColumnSlice staticSlice = new ColumnSlice(ByteBufferUtil.EMPTY_BYTE_BUFFER, baseCfs.metadata.getStaticColumnNameBuilder().buildAsEndOfRange()); - slices = new ColumnSlice[]{ staticSlice, dataSlice }; - } - else - { - slices = new ColumnSlice[]{ dataSlice }; - } - SliceQueryFilter dataFilter = new SliceQueryFilter(slices, false, Integer.MAX_VALUE, baseCfs.metadata.clusteringKeyColumns().size()); ++ ColumnSlice dataSlice = new ColumnSlice(start, entry.indexedEntryPrefix.end()); ++ // If the table has static columns, we must fetch them too as they may need to be returned too. ++ // Note that this is potentially wasteful for 2 reasons: ++ // 1) we will retrieve the static parts for each indexed row, even if we have more than one row in ++ // the same partition. If we were to group data queries to rows on the same slice, which would ++ // speed up things in general, we would also optimize here since we would fetch static columns only ++ // once for each group. ++ // 2) at this point we don't know if the user asked for static columns or not, so we might be fetching ++ // them for nothing. We would however need to ship the list of "CQL3 columns selected" with getRangeSlice ++ // to be able to know that. ++ // TODO: we should improve both point above ++ ColumnSlice[] slices = baseCfs.metadata.hasStaticColumns() ++ ? new ColumnSlice[]{ baseCfs.metadata.comparator.staticPrefix().slice(), dataSlice } ++ : new ColumnSlice[]{ dataSlice }; ++ SliceQueryFilter dataFilter = new SliceQueryFilter(slices, false, Integer.MAX_VALUE, baseCfs.metadata.clusteringColumns().size()); ColumnFamily newData = baseCfs.getColumnFamily(new QueryFilter(dk, baseCfs.name, dataFilter, filter.timestamp)); if (newData == null || index.isStale(entry, newData, filter.timestamp)) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java index be66d21,9250b0f..3184741 --- a/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractCompositeType.java @@@ -34,15 -34,23 +34,23 @@@ import java.util.List */ public abstract class AbstractCompositeType extends AbstractType<ByteBuffer> { + // changes bb position - protected static int getShortLength(ByteBuffer bb) + public static int getShortLength(ByteBuffer bb) { int length = (bb.get() & 0xFF) << 8; return length | (bb.get() & 0xFF); } + // Doesn't change bb position + protected static int getShortLength(ByteBuffer bb, int position) + { + int length = (bb.get(position) & 0xFF) << 8; + return length | (bb.get(position + 1) & 0xFF); + } + // changes bb position - protected static void putShortLength(ByteBuffer bb, int length) + public static void putShortLength(ByteBuffer bb, int length) { bb.put((byte) ((length >> 8) & 0xFF)); bb.put((byte) (length & 0xFF)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/marshal/CompositeType.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/db/marshal/DynamicCompositeType.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/63b1ef4e/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java index 25ffc29,65e3be1..af88853 --- a/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java +++ b/src/java/org/apache/cassandra/hadoop/pig/AbstractCassandraStorage.java @@@ -25,8 -25,8 +25,9 @@@ import java.nio.ByteBuffer import java.nio.charset.CharacterCodingException; import java.util.*; + import com.google.common.collect.Iterables; +import org.apache.cassandra.db.Cell; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.auth.IAuthenticator; @@@ -675,12 -674,12 +676,12 @@@ public abstract class AbstractCassandra return columnDefs; // otherwise for CqlStorage, check metadata for classic thrift tables - CFDefinition cfDefinition = getCfDefinition(keyspace, column_family, client); - for (CFDefinition.Name column : Iterables.concat(cfDefinition.staticColumns(), cfDefinition.regularColumns())) + CFMetaData cfm = getCFMetaData(keyspace, column_family, client); - for (ColumnDefinition def : cfm.regularColumns()) ++ for (ColumnDefinition def : cfm.regularAndStaticColumns()) { ColumnDef cDef = new ColumnDef(); - String columnName = column.name.toString(); - String type = column.type.toString(); + String columnName = def.name.toString(); + String type = def.type.toString(); logger.debug("name: {}, type: {} ", columnName, type); cDef.name = ByteBufferUtil.bytes(columnName); cDef.validation_class = type;