Author: jbellis Date: Thu Jun 9 22:15:25 2011 New Revision: 1134109 URL: http://svn.apache.org/viewvc?rev=1134109&view=rev Log: merge from 0.7
Modified: cassandra/branches/cassandra-0.8/ (props changed) cassandra/branches/cassandra-0.8/CHANGES.txt cassandra/branches/cassandra-0.8/contrib/ (props changed) cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java Propchange: cassandra/branches/cassandra-0.8/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7:1026516-1133391 +/cassandra/branches/cassandra-0.7:1026516-1133874 /cassandra/branches/cassandra-0.7.0:1053690-1055654 /cassandra/branches/cassandra-0.8:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0:1125021-1130369 Modified: cassandra/branches/cassandra-0.8/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1134109&r1=1134108&r2=1134109&view=diff ============================================================================== --- cassandra/branches/cassandra-0.8/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.8/CHANGES.txt Thu Jun 9 22:15:25 2011 @@ -42,6 +42,8 @@ by nio sockets (CASSANDRA-2654) * restrict repair streaming to specific columnfamilies (CASSANDRA-2280) * fix nodetool ring use with Ec2Snitch (CASSANDRA-2733) + * fix removing columns and subcolumns that are supressed by a row or + supercolumn tombstone during replica resolution (CASSANDRA-2590) 0.8.0-final Propchange: cassandra/branches/cassandra-0.8/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009 -/cassandra/branches/cassandra-0.7/contrib:1026516-1133391 +/cassandra/branches/cassandra-0.7/contrib:1026516-1133874 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654 /cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369 Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654 /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369 Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654 /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369 Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654 /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369 Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654 /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369 Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Jun 9 22:15:25 2011 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133391 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133874 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654 /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125041 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369 Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java?rev=1134109&r1=1134108&r2=1134109&view=diff ============================================================================== --- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java (original) +++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java Thu Jun 9 22:15:25 2011 @@ -26,7 +26,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.commons.collections.iterators.CollatingIterator; + import org.apache.cassandra.db.*; +import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; @@ -122,19 +127,30 @@ public class RowRepairResolver extends A ColumnFamily resolved = null; for (ColumnFamily cf : versions) { - if (cf != null) - { - resolved = cf.cloneMe(); - break; - } + if (cf == null) + continue; + + if (resolved == null) + resolved = cf.cloneMeShallow(); + else + resolved.delete(cf); } if (resolved == null) return null; - for (ColumnFamily cf : versions) - resolved.resolve(cf); - - return resolved; + // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes. + // this will handle removing columns and subcolumns that are supressed by a row or + // supercolumn tombstone. + QueryFilter filter = new QueryFilter(null, new QueryPath(resolved.metadata().cfName), new IdentityQueryFilter()); + CollatingIterator iter = new CollatingIterator(resolved.metadata().comparator.columnComparator); + for (ColumnFamily version : versions) + { + if (version == null) + continue; + iter.addIterator(version.getColumnsMap().values().iterator()); + } + filter.collectCollatedColumns(resolved, iter, Integer.MIN_VALUE); + return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE); } public Row getData() throws IOException Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java?rev=1134109&r1=1134108&r2=1134109&view=diff ============================================================================== --- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java (original) +++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java Thu Jun 9 22:15:25 2011 @@ -58,6 +58,14 @@ public class Util return new Column(ByteBufferUtil.bytes(name), ByteBufferUtil.bytes(value), timestamp); } + public static SuperColumn superColumn(ColumnFamily cf, String name, Column... columns) + { + SuperColumn sc = new SuperColumn(ByteBufferUtil.bytes(name), cf.metadata().comparator); + for (Column c : columns) + sc.addColumn(c); + return sc; + } + public static Token token(String key) { return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key)); Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java?rev=1134109&r1=1134108&r2=1134109&view=diff ============================================================================== --- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java (original) +++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java Thu Jun 9 22:15:25 2011 @@ -511,7 +511,18 @@ public class TableTest extends CleanupHe public static void assertColumns(ColumnFamily cf, String... columnNames) { - Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns(); + assertColumns((IColumnContainer)cf, columnNames); + } + + public static void assertSubColumns(ColumnFamily cf, String scName, String... columnNames) + { + IColumnContainer sc = cf == null ? null : ((IColumnContainer)cf.getColumn(ByteBufferUtil.bytes(scName))); + assertColumns(sc, columnNames); + } + + public static void assertColumns(IColumnContainer container, String... columnNames) + { + Collection<IColumn> columns = container == null ? new TreeSet<IColumn>() : container.getSortedColumns(); List<String> L = new ArrayList<String>(); for (IColumn column : columns) { @@ -540,9 +551,28 @@ public class TableTest extends CleanupHe assert Arrays.equals(la, columnNames1) : String.format("Columns [%s(as string: %s)])] is not expected [%s]", - ((cf == null) ? "" : cf.getComparator().getColumnsString(columns)), + ((container == null) ? "" : container.getComparator().getColumnsString(columns)), lasb.toString(), StringUtils.join(columnNames1, ",")); } + public static void assertColumn(ColumnFamily cf, String name, String value, long timestamp) + { + assertColumn(cf.getColumn(ByteBufferUtil.bytes(name)), value, timestamp); + } + + public static void assertSubColumn(ColumnFamily cf, String scName, String name, String value, long timestamp) + { + SuperColumn sc = (SuperColumn)cf.getColumn(ByteBufferUtil.bytes(scName)); + assertColumn(sc.getSubColumn(ByteBufferUtil.bytes(name)), value, timestamp); + } + + public static void assertColumn(IColumn column, String value, long timestamp) + { + assertNotNull(column); + assertEquals(0, ByteBufferUtil.compareUnsigned(column.value(), ByteBufferUtil.bytes(value))); + assertEquals(timestamp, column.timestamp()); + } + + } Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java?rev=1134109&r1=1134108&r2=1134109&view=diff ============================================================================== --- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java (original) +++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java Thu Jun 9 22:15:25 2011 @@ -23,14 +23,16 @@ package org.apache.cassandra.service; import java.util.Arrays; -import org.apache.cassandra.SchemaLoader; import org.junit.Test; +import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.ColumnFamily; +import org.apache.cassandra.db.SuperColumn; -import static org.apache.cassandra.db.TableTest.assertColumns; +import static junit.framework.Assert.*; import static org.apache.cassandra.Util.column; -import static junit.framework.Assert.assertNull; +import static org.apache.cassandra.Util.superColumn; +import static org.apache.cassandra.db.TableTest.*; public class RowResolverTest extends SchemaLoader { @@ -93,4 +95,110 @@ public class RowResolverTest extends Sch { assertNull(RowRepairResolver.resolveSuperset(Arrays.<ColumnFamily>asList(null, null))); } + + @Test + public void testResolveDeleted() + { + // one CF with columns timestamped before a delete in another cf + ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1"); + cf1.addColumn(column("one", "A", 0)); + + ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1"); + cf2.delete((int) (System.currentTimeMillis() / 1000), 1); + + ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2)); + // no columns in the cf + assertColumns(resolved); + assertTrue(resolved.isMarkedForDelete()); + assertEquals(1, resolved.getMarkedForDeleteAt()); + + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + scf1.addColumn(superColumn(scf1, "super-foo", column("one", "A", 0))); + + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.delete((int) (System.currentTimeMillis() / 1000), 1); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2)); + // no columns in the cf + assertColumns(superResolved); + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(1, superResolved.getMarkedForDeleteAt()); + } + + @Test + public void testResolveDeletedSuper() + { + // subcolumn is newer than a tombstone on its parent, but not newer than the row deletion + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + SuperColumn sc = superColumn(scf1, "super-foo", column("one", "A", 1)); + sc.markForDeleteAt((int) (System.currentTimeMillis() / 1000), 0); + scf1.addColumn(sc); + + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2)); + // no columns in the cf + assertColumns(superResolved); + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(2, superResolved.getMarkedForDeleteAt()); + } + + @Test + public void testResolveMultipleDeleted() + { + // deletes and columns with interleaved timestamp, with out of order return sequence + + ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1"); + cf1.delete((int) (System.currentTimeMillis() / 1000), 0); + + // these columns created after the previous deletion + ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1"); + cf2.addColumn(column("one", "A", 1)); + cf2.addColumn(column("two", "A", 1)); + + //this column created after the next delete + ColumnFamily cf3 = ColumnFamily.create("Keyspace1", "Standard1"); + cf3.addColumn(column("two", "B", 3)); + + ColumnFamily cf4 = ColumnFamily.create("Keyspace1", "Standard1"); + cf4.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2, cf3, cf4)); + // will have deleted marker and one column + assertColumns(resolved, "two"); + assertColumn(resolved, "two", "B", 3); + assertTrue(resolved.isMarkedForDelete()); + assertEquals(2, resolved.getMarkedForDeleteAt()); + + + ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1"); + scf1.delete((int) (System.currentTimeMillis() / 1000), 0); + + // these columns created after the previous deletion + ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1"); + scf2.addColumn(superColumn(scf2, "super1", column("one", "A", 1), column("two", "A", 1))); + + //these columns created after the next delete + ColumnFamily scf3 = ColumnFamily.create("Keyspace1", "Super1"); + scf3.addColumn(superColumn(scf3, "super1", column("two", "B", 3))); + scf3.addColumn(superColumn(scf3, "super2", column("three", "A", 3), column("four", "A", 3))); + + ColumnFamily scf4 = ColumnFamily.create("Keyspace1", "Super1"); + scf4.delete((int) (System.currentTimeMillis() / 1000), 2); + + ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2, scf3, scf4)); + // will have deleted marker and two super cols + assertColumns(superResolved, "super1", "super2"); + + assertSubColumns(superResolved, "super1", "two"); + assertSubColumn(superResolved, "super1", "two", "B", 3); + + assertSubColumns(superResolved, "super2", "four", "three"); + assertSubColumn(superResolved, "super2", "three", "A", 3); + assertSubColumn(superResolved, "super2", "four", "A", 3); + + assertTrue(superResolved.isMarkedForDelete()); + assertEquals(2, superResolved.getMarkedForDeleteAt()); + } }