Author: jbellis Date: Fri Jun 10 20:19:23 2011 New Revision: 1134431 URL: http://svn.apache.org/viewvc?rev=1134431&view=rev Log: merge from 0.8
Removed: cassandra/trunk/examples/bmt/CassandraBulkLoader.java cassandra/trunk/examples/bmt/README.txt Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/contrib/ (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g cassandra/trunk/src/java/org/apache/cassandra/db/Table.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java cassandra/trunk/src/java/org/apache/cassandra/service/RowRepairResolver.java cassandra/trunk/test/system/test_cql.py cassandra/trunk/test/unit/org/apache/cassandra/Util.java cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java cassandra/trunk/test/unit/org/apache/cassandra/service/RowResolverTest.java cassandra/trunk/tools/stress/build.xml Propchange: cassandra/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7:1026516-1133391 +/cassandra/branches/cassandra-0.7:1026516-1133874 /cassandra/branches/cassandra-0.7.0:1053690-1055654 -/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0:1125021-1130369 /cassandra/branches/cassandra-0.8.1:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Fri Jun 10 20:19:23 2011 @@ -8,6 +8,7 @@ - improve JDBC spec compliance (CASSANDRA-2720) - ALTER COLUMNFAMILY (CASSANDRA-1709) - DROP INDEX (CASSANDRA-2617) + - add SCHEMA/TABLE as aliases for KS/CF (CASSANDRA-2743) * add support for comparator parameters and a generic ReverseType (CASSANDRA-2355) * add CompositeType and DynamicCompositeType (CASSANDRA-2231) @@ -45,6 +46,9 @@ by nio sockets (CASSANDRA-2654) * restrict repair streaming to specific columnfamilies (CASSANDRA-2280) * fix nodetool ring use with Ec2Snitch (CASSANDRA-2733) + * fix removing columns and subcolumns that are supressed by a row or + supercolumn tombstone during replica resolution (CASSANDRA-2590) + * support sstable2json against snapshot sstables (CASSANDRA-2386) 0.8.0-final Propchange: cassandra/trunk/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009 -/cassandra/branches/cassandra-0.7/contrib:1026516-1133391 +/cassandra/branches/cassandra-0.7/contrib:1026516-1133874 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654 -/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jun 10 20:19:23 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1133945 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1134430 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689 Modified: cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g (original) +++ cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g Fri Jun 10 20:19:23 2011 @@ -498,8 +498,10 @@ K_TRUNCATE: T R U N C A T E; K_DELETE: D E L E T E; K_IN: I N; K_CREATE: C R E A T E; -K_KEYSPACE: K E Y S P A C E; -K_COLUMNFAMILY: C O L U M N F A M I L Y; +K_KEYSPACE: ( K E Y S P A C E + | S C H E M A ); +K_COLUMNFAMILY:( C O L U M N F A M I L Y + | T A B L E ); K_INDEX: I N D E X; K_ON: O N; K_DROP: D R O P; Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Table.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Fri Jun 10 20:19:23 2011 @@ -55,8 +55,9 @@ public class Table { public static final String SYSTEM_TABLE = "system"; + public static final String SNAPSHOT_SUBDIR_NAME = "snapshots"; + private static final Logger logger = LoggerFactory.getLogger(Table.class); - private static final String SNAPSHOT_SUBDIR_NAME = "snapshots"; /** * accesses to CFS.memtable should acquire this for thread safety. Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Descriptor.java Fri Jun 10 20:19:23 2011 @@ -26,6 +26,7 @@ import java.util.StringTokenizer; import com.google.common.base.Objects; +import org.apache.cassandra.db.Table; import org.apache.cassandra.utils.Pair; /** @@ -131,7 +132,7 @@ public class Descriptor public static Pair<Descriptor,String> fromFilename(File directory, String name) { // name of parent directory is keyspace name - String ksname = directory.getName(); + String ksname = extractKeyspaceName(directory); // tokenize the filename StringTokenizer st = new StringTokenizer(name, "-"); @@ -165,6 +166,43 @@ public class Descriptor } /** + * Extracts the keyspace name out of the directory name. Snapshot directories have a slightly different + * path structure and need to be treated differently. + * + * Regular path: "<ksname>/<cfname>-[tmp-][<version>-]<gen>-<component>" + * Snapshot path: "<ksname>/snapshots/<snapshot-name>/<cfname>-[tmp-][<version>-]<gen>-<component>" + * + * @param directory a directory containing SSTables + * @return the keyspace name + */ + public static String extractKeyspaceName(File directory) { + + if (isSnapshotInPath(directory)) + { + // We need to move backwards. If this is a snapshot, first parent takes us to: + // <ksname>/snapshots/ and second call to parent takes us to <ksname>. + return directory.getParentFile().getParentFile().getName(); + } + return directory.getName(); + } + + /** + * @return <code>TRUE</code> if this directory represents a snapshot directory. <code>FALSE</code> otherwise. + */ + private static boolean isSnapshotInPath(File directory) { + File curDirectory = directory; + while (curDirectory != null) + { + if (curDirectory.getName().equals(Table.SNAPSHOT_SUBDIR_NAME)) + return true; + curDirectory = curDirectory.getParentFile(); + } + + // The directory does not represent a snapshot directory. + return false; + } + + /** * @return A clone of this descriptor with the given 'temporary' status. */ public Descriptor asTemporary(boolean temporary) Modified: cassandra/trunk/src/java/org/apache/cassandra/service/RowRepairResolver.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/RowRepairResolver.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/RowRepairResolver.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/RowRepairResolver.java Fri Jun 10 20:19:23 2011 @@ -26,7 +26,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.commons.collections.iterators.CollatingIterator; + import org.apache.cassandra.db.*; +import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; @@ -122,19 +127,30 @@ public class RowRepairResolver extends A ColumnFamily resolved = null; for (ColumnFamily cf : versions) { - if (cf != null) - { - resolved = cf.cloneMe(); - break; - } + if (cf == null) + continue; + + if (resolved == null) + resolved = cf.cloneMeShallow(); + else + resolved.delete(cf); } if (resolved == null) return null; - for (ColumnFamily cf : versions) - resolved.resolve(cf); - - return resolved; + // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes. + // this will handle removing columns and subcolumns that are supressed by a row or + // supercolumn tombstone. + QueryFilter filter = new QueryFilter(null, new QueryPath(resolved.metadata().cfName), new IdentityQueryFilter()); + CollatingIterator iter = new CollatingIterator(resolved.metadata().comparator.columnComparator); + for (ColumnFamily version : versions) + { + if (version == null) + continue; + iter.addIterator(version.getColumnsMap().values().iterator()); + } + filter.collectCollatedColumns(resolved, iter, Integer.MIN_VALUE); + return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE); } public Row getData() throws IOException Modified: cassandra/trunk/test/system/test_cql.py URL: http://svn.apache.org/viewvc/cassandra/trunk/test/system/test_cql.py?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/test/system/test_cql.py (original) +++ cassandra/trunk/test/system/test_cql.py Fri Jun 10 20:19:23 2011 @@ -44,7 +44,7 @@ def load_sample(dbconn): WITH comparator = ascii AND default_validation = ascii; """) dbconn.execute(""" - CREATE COLUMNFAMILY StandardString2 (KEY text PRIMARY KEY) + CREATE TABLE StandardString2 (KEY text PRIMARY KEY) WITH comparator = ascii AND default_validation = ascii; """) dbconn.execute(""" @@ -56,7 +56,7 @@ def load_sample(dbconn): WITH comparator = bigint AND default_validation = ascii; """) dbconn.execute(""" - CREATE COLUMNFAMILY StandardIntegerA (KEY text PRIMARY KEY) + CREATE TABLE StandardIntegerA (KEY text PRIMARY KEY) WITH comparator = varint AND default_validation = ascii; """) dbconn.execute(""" @@ -76,7 +76,7 @@ def load_sample(dbconn): WITH comparator = ascii AND default_validation = ascii; """) dbconn.execute(""" - CREATE COLUMNFAMILY CounterCF (KEY text PRIMARY KEY, count_me counter) + CREATE TABLE CounterCF (KEY text PRIMARY KEY, count_me counter) WITH comparator = ascii AND default_validation = counter; """) dbconn.execute("CREATE INDEX ON IndexedA (birthdate)") @@ -414,7 +414,7 @@ class TestCql(ThriftTester): "create a new keyspace" cursor = init() cursor.execute(""" - CREATE KEYSPACE TestKeyspace42 WITH strategy_options:DC1 = '1' + CREATE SCHEMA TestKeyspace42 WITH strategy_options:DC1 = '1' AND strategy_class = 'NetworkTopologyStrategy' """) @@ -436,7 +436,7 @@ class TestCql(ThriftTester): # TODO: temporary (until this can be done with CQL). thrift_client.describe_keyspace("Keyspace4Drop") - cursor.execute('DROP KEYSPACE Keyspace4Drop;') + cursor.execute('DROP SCHEMA Keyspace4Drop;') # Technically this should throw a ttypes.NotFound(), but this is # temporary and so not worth requiring it on PYTHONPATH. @@ -448,7 +448,7 @@ class TestCql(ThriftTester): "create a new column family" cursor = init() cursor.execute(""" - CREATE KEYSPACE CreateCFKeyspace WITH strategy_options:replication_factor = '1' + CREATE SCHEMA CreateCFKeyspace WITH strategy_options:replication_factor = '1' AND strategy_class = 'SimpleStrategy'; """) cursor.execute("USE CreateCFKeyspace;") Modified: cassandra/trunk/test/unit/org/apache/cassandra/Util.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/Util.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/Util.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/Util.java Fri Jun 10 20:19:23 2011 @@ -58,6 +58,14 @@ public class Util return new Column(ByteBufferUtil.bytes(name), ByteBufferUtil.bytes(value), timestamp); } + public static SuperColumn superColumn(ColumnFamily cf, String name, Column... columns) + { + SuperColumn sc = new SuperColumn(ByteBufferUtil.bytes(name), cf.metadata().comparator); + for (Column c : columns) + sc.addColumn(c); + return sc; + } + public static Token token(String key) { return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key)); Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java Fri Jun 10 20:19:23 2011 @@ -511,7 +511,18 @@ public class TableTest extends CleanupHe public static void assertColumns(ColumnFamily cf, String... columnNames) { - Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns(); + assertColumns((IColumnContainer)cf, columnNames); + } + + public static void assertSubColumns(ColumnFamily cf, String scName, String... columnNames) + { + IColumnContainer sc = cf == null ? null : ((IColumnContainer)cf.getColumn(ByteBufferUtil.bytes(scName))); + assertColumns(sc, columnNames); + } + + public static void assertColumns(IColumnContainer container, String... columnNames) + { + Collection<IColumn> columns = container == null ? new TreeSet<IColumn>() : container.getSortedColumns(); List<String> L = new ArrayList<String>(); for (IColumn column : columns) { @@ -540,9 +551,28 @@ public class TableTest extends CleanupHe assert Arrays.equals(la, columnNames1) : String.format("Columns [%s(as string: %s)])] is not expected [%s]", - ((cf == null) ? "" : cf.getComparator().getColumnsString(columns)), + ((container == null) ? "" : container.getComparator().getColumnsString(columns)), lasb.toString(), StringUtils.join(columnNames1, ",")); } + public static void assertColumn(ColumnFamily cf, String name, String value, long timestamp) + { + assertColumn(cf.getColumn(ByteBufferUtil.bytes(name)), value, timestamp); + } + + public static void assertSubColumn(ColumnFamily cf, String scName, String name, String value, long timestamp) + { + SuperColumn sc = (SuperColumn)cf.getColumn(ByteBufferUtil.bytes(scName)); + assertColumn(sc.getSubColumn(ByteBufferUtil.bytes(name)), value, timestamp); + } + + public static void assertColumn(IColumn column, String value, long timestamp) + { + assertNotNull(column); + assertEquals(0, ByteBufferUtil.compareUnsigned(column.value(), ByteBufferUtil.bytes(value))); + assertEquals(timestamp, column.timestamp()); + } + + } Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java Fri Jun 10 20:19:23 2011 @@ -25,6 +25,8 @@ import java.io.File; import org.junit.Test; +import org.apache.cassandra.db.Table; + public class DescriptorTest { @Test @@ -34,4 +36,28 @@ public class DescriptorTest assert descriptor.version.equals(Descriptor.LEGACY_VERSION); assert descriptor.usesOldBloomFilter; } + + @Test + public void testExtractKeyspace() + { + // Test a path representing a SNAPSHOT directory + String dirPath = "Keyspace10" + File.separator + Table.SNAPSHOT_SUBDIR_NAME + File.separator + System.currentTimeMillis(); + assertKeyspace("Keyspace10", dirPath); + + // Test a path representing a regular SSTables directory + dirPath = "Keyspace11"; + assertKeyspace("Keyspace11", dirPath); + } + + private void assertKeyspace(String expectedKsName, String dirPath) { + File dir = new File(dirPath); + dir.deleteOnExit(); + + // Create and check. + if (!dir.mkdirs()) + throw new RuntimeException("Unable to create directories:" + dirPath); + + String currentKsName = Descriptor.extractKeyspaceName(dir); + assert expectedKsName.equals(currentKsName); + } } Modified: cassandra/trunk/test/unit/org/apache/cassandra/service/RowResolverTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/service/RowResolverTest.java?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/service/RowResolverTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/service/RowResolverTest.java Fri Jun 10 20:19:23 2011 @@ -23,14 +23,16 @@ package org.apache.cassandra.service; import java.util.Arrays; -import org.apache.cassandra.SchemaLoader; import org.junit.Test; +import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.SuperColumn; -import static org.apache.cassandra.db.TableTest.assertColumns; +import static junit.framework.Assert.*; import static org.apache.cassandra.Util.column; -import static junit.framework.Assert.assertNull; +import static org.apache.cassandra.Util.superColumn; +import static org.apache.cassandra.db.TableTest.*; public class RowResolverTest extends SchemaLoader { @@ -93,4 +95,110 @@ public class RowResolverTest extends Sch { assertNull(RowRepairResolver.resolveSuperset(Arrays.<ColumnFamily>asList(null, null))); } + + @Test + public void testResolveDeleted() + { + // one CF with columns timestamped before a delete in another cf + ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1"); + cf1.addColumn(column("one", "A", 0)); + + ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1"); + cf2.delete((int) (System.currentTimeMillis() / 1000), 1); + + ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2)); + // no columns in the cf + assertColumns(resolved); + assertTrue(resolved.isMarkedForDelete()); + assertEquals(1, resolved.getMarkedForDeleteAt()); + + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + scf1.addColumn(superColumn(scf1, "super-foo", column("one", "A", 0))); + + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.delete((int) (System.currentTimeMillis() / 1000), 1); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2)); + // no columns in the cf + assertColumns(superResolved); + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(1, superResolved.getMarkedForDeleteAt()); + } + + @Test + public void testResolveDeletedSuper() + { + // subcolumn is newer than a tombstone on its parent, but not newer than the row deletion + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + SuperColumn sc = superColumn(scf1, "super-foo", column("one", "A", 1)); + sc.markForDeleteAt((int) (System.currentTimeMillis() / 1000), 0); + scf1.addColumn(sc); + + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2)); + // no columns in the cf + assertColumns(superResolved); + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(2, superResolved.getMarkedForDeleteAt()); + } + + @Test + public void testResolveMultipleDeleted() + { + // deletes and columns with interleaved timestamp, with out of order return sequence + + ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1"); + cf1.delete((int) (System.currentTimeMillis() / 1000), 0); + + // these columns created after the previous deletion + ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1"); + cf2.addColumn(column("one", "A", 1)); + cf2.addColumn(column("two", "A", 1)); + + //this column created after the next delete + ColumnFamily cf3 = ColumnFamily.create("Keyspace1", "Standard1"); + cf3.addColumn(column("two", "B", 3)); + + ColumnFamily cf4 = ColumnFamily.create("Keyspace1", "Standard1"); + cf4.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2, cf3, cf4)); + // will have deleted marker and one column + assertColumns(resolved, "two"); + assertColumn(resolved, "two", "B", 3); + assertTrue(resolved.isMarkedForDelete()); + assertEquals(2, resolved.getMarkedForDeleteAt()); + + + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + scf1.delete((int) (System.currentTimeMillis() / 1000), 0); + + // these columns created after the previous deletion + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.addColumn(superColumn(scf2, "super1", column("one", "A", 1), column("two", "A", 1))); + + //these columns created after the next delete + ColumnFamily scf3 = ColumnFamily.create("Keyspace1", "Super1"); + scf3.addColumn(superColumn(scf3, "super1", column("two", "B", 3))); + scf3.addColumn(superColumn(scf3, "super2", column("three", "A", 3), column("four", "A", 3))); + + ColumnFamily scf4 = ColumnFamily.create("Keyspace1", "Super1"); + scf4.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2, scf3, scf4)); + // will have deleted marker and two super cols + assertColumns(superResolved, "super1", "super2"); + + assertSubColumns(superResolved, "super1", "two"); + assertSubColumn(superResolved, "super1", "two", "B", 3); + + assertSubColumns(superResolved, "super2", "four", "three"); + assertSubColumn(superResolved, "super2", "three", "A", 3); + assertSubColumn(superResolved, "super2", "four", "A", 3); + + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(2, superResolved.getMarkedForDeleteAt()); + } } Modified: cassandra/trunk/tools/stress/build.xml URL: http://svn.apache.org/viewvc/cassandra/trunk/tools/stress/build.xml?rev=1134431&r1=1134430&r2=1134431&view=diff ============================================================================== --- cassandra/trunk/tools/stress/build.xml (original) +++ cassandra/trunk/tools/stress/build.xml Fri Jun 10 20:19:23 2011 @@ -17,7 +17,7 @@ ~ specific language governing permissions and limitations ~ under the License. --> -<project basedir="." default="build" name="stress"> +<project basedir="." default="jar" name="stress"> <property name="cassandra.dir" value="../.." /> <property name="cassandra.lib" value="${cassandra.dir}/lib" /> <property name="build.src" value="${basedir}/src" /> @@ -49,9 +49,19 @@ </target> <target name="jar" depends="build"> + <manifest file="MANIFEST.MF"> + <attribute name="Built-By" value="Pavel Yaskevich"/> + <attribute name="Main-Class" value="org.apache.cassandra.stress.Stress"/> + </manifest> + <mkdir dir="${build.classes}/META-INF" /> - <jar jarfile="${build.out}/${final.name}.jar" - basedir="${build.classes}" /> + + <jar destfile="${build.out}/${final.name}.jar" manifest="MANIFEST.MF"> + <fileset dir="${build.classes}"/> + <fileset dir="${cassandra.dir}/build/classes/main" /> + <fileset dir="${cassandra.dir}/build/classes/thrift" /> + <zipgroupfileset dir="${cassandra.lib}" includes="*.jar" /> + </jar> </target> <target name="clean">