Author: jbellis Date: Mon May 10 14:59:13 2010 New Revision: 942768 URL: http://svn.apache.org/viewvc?rev=942768&view=rev Log: merge from 0.6
Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java Propchange: cassandra/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6:922689-942206 +/cassandra/branches/cassandra-0.6:922689-942757 /incubator/cassandra/branches/cassandra-0.3:774578-796573 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5:888872-915439 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=942768&r1=942767&r2=942768&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Mon May 10 14:59:13 2010 @@ -36,6 +36,7 @@ dev (CASSANDRA-1060) * disable GCInspector on non-Sun JVMs (CASSANDRA-1061) * fix tombstone handling in sstable rows with no other data (CASSANDRA-1063) + * fix size of row in spanned index entries (CASSANDRA-1056) 0.6.1 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942206 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942757 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942206 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942757 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942206 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942757 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942206 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942757 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon May 10 14:59:13 2010 @@ -1,4 +1,4 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942206 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942757 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502 Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?rev=942768&r1=942767&r2=942768&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java Mon May 10 14:59:13 2010 @@ -41,11 +41,11 @@ public class IndexSummary private int keysWritten = 0; private long lastIndexPosition; - public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long dataSize, long indexPosition, long nextIndexPosition) + public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long rowSize, long indexPosition, long nextIndexPosition) { boolean spannedIndexEntry = DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap && RowIndexedReader.bufferIndex(indexPosition) != RowIndexedReader.bufferIndex(nextIndexPosition); - if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry) + if ((keysWritten++ % INDEX_INTERVAL == 0) || spannedIndexEntry) { if (indexPositions == null) { @@ -61,13 +61,18 @@ public class IndexSummary spannedIndexDataPositions = new HashMap<KeyPosition, SSTable.PositionSize>(); spannedIndexPositions = new HashMap<Long, KeyPosition>(); } - spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition, dataSize)); + spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition, rowSize)); spannedIndexPositions.put(info.indexPosition, info); } } lastIndexPosition = indexPosition; } + public Map<KeyPosition, SSTable.PositionSize> getSpannedIndexDataPositions() + { + return spannedIndexDataPositions; + } + public List<KeyPosition> getIndexPositions() { return indexPositions; Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=942768&r1=942767&r2=942768&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Mon May 10 14:59:13 2010 @@ -59,7 +59,6 @@ public class SSTableWriter extends SSTab { private static Logger logger = LoggerFactory.getLogger(SSTableWriter.class); - private long keysWritten; private BufferedRandomAccessFile dataFile; private BufferedRandomAccessFile indexFile; private DecoratedKey lastWrittenKey; @@ -90,7 +89,7 @@ public class SSTableWriter extends SSTab return (lastWrittenKey == null) ? 0 : dataFile.getFilePointer(); } - private void afterAppend(DecoratedKey decoratedKey, long dataPosition, int dataSize) throws IOException + private void afterAppend(DecoratedKey decoratedKey, long dataPosition) throws IOException { byte[] diskKey = partitioner.convertToDiskFormat(decoratedKey); bf.add(diskKey); @@ -103,7 +102,8 @@ public class SSTableWriter extends SSTab if (logger.isTraceEnabled()) logger.trace("wrote index of " + decoratedKey + " at " + indexPosition); - indexSummary.maybeAddEntry(decoratedKey, dataPosition, dataSize, indexPosition, indexFile.getFilePointer()); + int rowSize = (int)(dataFile.getFilePointer() - dataPosition); + indexSummary.maybeAddEntry(decoratedKey, dataPosition, rowSize, indexPosition, indexFile.getFilePointer()); } // TODO make this take a DataOutputStream and wrap the byte[] version to combine them @@ -115,7 +115,7 @@ public class SSTableWriter extends SSTab assert length > 0; dataFile.writeInt(length); dataFile.write(buffer.getData(), 0, length); - afterAppend(decoratedKey, currentPosition, length); + afterAppend(decoratedKey, currentPosition); } public void append(DecoratedKey decoratedKey, byte[] value) throws IOException @@ -125,7 +125,7 @@ public class SSTableWriter extends SSTab assert value.length > 0; dataFile.writeInt(value.length); dataFile.write(value); - afterAppend(decoratedKey, currentPosition, value.length); + afterAppend(decoratedKey, currentPosition); } public SSTableReader closeAndOpenReader() throws IOException Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java?rev=942768&r1=942767&r2=942768&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java Mon May 10 14:59:13 2010 @@ -2,6 +2,7 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.concurrent.ExecutionException; +import java.util.Map; import org.junit.Test; @@ -9,22 +10,26 @@ import org.apache.cassandra.CleanupHelpe import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.io.util.BufferedRandomAccessFile; import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.Util; +import static org.junit.Assert.assertEquals; + public class SSTableReaderTest extends CleanupHelper { @Test public void testSpannedIndexPositions() throws IOException, ExecutionException, InterruptedException { - RowIndexedReader.BUFFER_SIZE = 40; + RowIndexedReader.BUFFER_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of spanned entries Table table = Table.open("Keyspace1"); ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); - // insert a bunch of data + // insert a bunch of data and compact to a single sstable CompactionManager.instance.disableAutoCompaction(); for (int j = 0; j < 100; j += 2) { @@ -52,5 +57,22 @@ public class SSTableReaderTest extends C DecoratedKey dk = Util.dk(String.valueOf(j)); assert sstable.getPosition(dk) == null; } + + // check positionsize information + assert sstable.indexSummary.getSpannedIndexDataPositions().entrySet().size() > 0; + for (Map.Entry<IndexSummary.KeyPosition, SSTable.PositionSize> entry : sstable.indexSummary.getSpannedIndexDataPositions().entrySet()) + { + IndexSummary.KeyPosition kp = entry.getKey(); + SSTable.PositionSize info = entry.getValue(); + + long nextIndexPosition = kp.indexPosition + 2 + StorageService.getPartitioner().convertToDiskFormat(kp.key).length + 8; + BufferedRandomAccessFile indexFile = new BufferedRandomAccessFile(sstable.indexFilename(), "r"); + indexFile.seek(nextIndexPosition); + String nextKey = indexFile.readUTF(); + + BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r"); + file.seek(info.position + info.size); + assertEquals(nextKey, file.readUTF()); + } } }