Author: jbellis Date: Tue Jan 11 14:09:36 2011 New Revision: 1057641 URL: http://svn.apache.org/viewvc?rev=1057641&view=rev Log: optimize supercolumn deserialization patch by Cliff Moon and jbellis; reviewed by Stu Hood for CASSANDRA-1891
Modified: cassandra/branches/cassandra-0.7/CHANGES.txt cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java Modified: cassandra/branches/cassandra-0.7/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1057641&r1=1057640&r2=1057641&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.7/CHANGES.txt Tue Jan 11 14:09:36 2011 @@ -2,18 +2,19 @@ * buffer network stack to avoid inefficient small TCP messages while avoiding the nagle/delayed ack problem (CASSANDRA-1896) * check log4j configuration for changes every 10s (CASSANDRA-1525, 1907) - * More-efficient cross-DC replication (CASSANDRA-1530) + * more-efficient cross-DC replication (CASSANDRA-1530) * upgrade to TFastFramedTransport (CASSANDRA-1743) * avoid polluting page cache with commitlog or sstable writes and seq scan operations (CASSANDRA-1470) * add RMI authentication options to nodetool (CASSANDRA-1921) - * Make snitches configurable at runtime (CASSANDRA-1374) + * make snitches configurable at runtime (CASSANDRA-1374) * retry hadoop split requests on connection failure (CASSANDRA-1927) * implement describeOwnership for BOP, COPP (CASSANDRA-1928) * make read repair behave as expected for ConsistencyLevel > ONE (CASSANDRA-982) * distributed test harness (CASSANDRA-1859) * reduce flush lock contention (CASSANDRA-1930) + * optimize supercolumn deserialization (CASSANDRA-1891) 0.7.0-dev Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java?rev=1057641&r1=1057640&r2=1057641&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java (original) +++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java Tue Jan 11 14:09:36 2011 @@ -213,7 +213,7 @@ public class Memtable implements Compara final Collection<IColumn> filteredColumns = filter.reversed ? cf.getReverseSortedColumns() : cf.getSortedColumns(); // ok to not have subcolumnComparator since we won't be adding columns to this object - IColumn startColumn = isSuper ? new SuperColumn(filter.start, null) : new Column(filter.start); + IColumn startColumn = isSuper ? new SuperColumn(filter.start, (AbstractType)null) : new Column(filter.start); Comparator<IColumn> comparator = filter.getColumnComparator(typeComparator); final PeekingIterator<IColumn> filteredIter = Iterators.peekingIterator(filteredColumns.iterator()); Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java?rev=1057641&r1=1057640&r2=1057641&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java (original) +++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java Tue Jan 11 14:09:36 2011 @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.ICompactSerializer2; +import org.apache.cassandra.io.util.ColumnSortedMap; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -41,7 +42,7 @@ import org.apache.cassandra.utils.FBUtil public class SuperColumn implements IColumn, IColumnContainer { - private static Logger logger_ = LoggerFactory.getLogger(SuperColumn.class); + private static Logger logger_ = LoggerFactory.getLogger(SuperColumn.class); public static SuperColumnSerializer serializer(AbstractType comparator) { @@ -58,7 +59,7 @@ public class SuperColumn implements ICol this(name, new ConcurrentSkipListMap<ByteBuffer, IColumn>(comparator)); } - private SuperColumn(ByteBuffer name, ConcurrentSkipListMap<ByteBuffer, IColumn> columns) + SuperColumn(ByteBuffer name, ConcurrentSkipListMap<ByteBuffer, IColumn> columns) { assert name != null; assert name.remaining() <= IColumn.MAX_NAME_LENGTH; @@ -312,6 +313,8 @@ public class SuperColumn implements ICol class SuperColumnSerializer implements ICompactSerializer2<IColumn> { + private static Logger logger = LoggerFactory.getLogger(SuperColumnSerializer.class); + private AbstractType comparator; public SuperColumnSerializer(AbstractType comparator) @@ -326,7 +329,7 @@ class SuperColumnSerializer implements I public void serialize(IColumn column, DataOutput dos) { - SuperColumn superColumn = (SuperColumn)column; + SuperColumn superColumn = (SuperColumn)column; FBUtilities.writeShortByteArray(column.name(), dos); try { @@ -349,21 +352,23 @@ class SuperColumnSerializer implements I public IColumn deserialize(DataInput dis) throws IOException { ByteBuffer name = FBUtilities.readShortByteArray(dis); - SuperColumn superColumn = new SuperColumn(name, comparator); int localDeleteTime = dis.readInt(); if (localDeleteTime != Integer.MIN_VALUE && localDeleteTime <= 0) { throw new IOException("Invalid localDeleteTime read: " + localDeleteTime); } - superColumn.markForDeleteAt(localDeleteTime, dis.readLong()); + long markedForDeleteAt = dis.readLong(); /* read the number of columns */ int size = dis.readInt(); - for ( int i = 0; i < size; ++i ) + ColumnSerializer serializer = Column.serializer(); + ColumnSortedMap preSortedMap = new ColumnSortedMap(comparator, serializer, dis, size); + SuperColumn superColumn = new SuperColumn(name, new ConcurrentSkipListMap<ByteBuffer,IColumn>(preSortedMap)); + if (localDeleteTime != Integer.MIN_VALUE && localDeleteTime <= 0) { - IColumn subColumn = Column.serializer().deserialize(dis); - superColumn.addColumn(subColumn); + throw new IOException("Invalid localDeleteTime read: " + localDeleteTime); } + superColumn.markForDeleteAt(localDeleteTime, markedForDeleteAt); return superColumn; } } Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java?rev=1057641&r1=1057640&r2=1057641&view=diff ============================================================================== --- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java (original) +++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java Tue Jan 11 14:09:36 2011 @@ -36,6 +36,7 @@ import org.apache.cassandra.io.sstable.D import org.apache.cassandra.io.sstable.SSTableReader; import static org.apache.cassandra.utils.FBUtilities.hexToBytes; import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile; +import static org.junit.Assert.assertEquals; import org.apache.cassandra.Util; @@ -82,6 +83,9 @@ public class SSTableImportTest extends S QueryFilter qf = QueryFilter.getNamesFilter(Util.dk("rowA"), new QueryPath("Super4", null, null), ByteBufferUtil.bytes("superA")); ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); IColumn superCol = cf.getColumn(ByteBufferUtil.bytes("superA")); - assert superCol.getSubColumn(ByteBufferUtil.bytes("colAA")).value().equals(ByteBuffer.wrap(hexToBytes("76616c75654141"))); + assert superCol != null; + assert superCol.getSubColumns().size() > 0; + IColumn subColumn = superCol.getSubColumn(ByteBufferUtil.bytes("colAA")); + assert subColumn.value().equals(ByteBuffer.wrap(hexToBytes("76616c75654141"))); } }