Author: jbellis Date: Wed Oct 5 18:33:38 2011 New Revision: 1179370 URL: http://svn.apache.org/viewvc?rev=1179370&view=rev Log: merge from 0.8
Modified: cassandra/branches/cassandra-1.0.0/ (props changed) cassandra/branches/cassandra-1.0.0/CHANGES.txt cassandra/branches/cassandra-1.0.0/contrib/ (props changed) cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java Propchange: cassandra/branches/cassandra-1.0.0/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0:1053690-1055654 -/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0:1125021-1130369 /cassandra/branches/cassandra-0.8.1:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original) +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Wed Oct 5 18:33:38 2011 @@ -17,6 +17,7 @@ * Fix broken CompressedRandomAccessReaderTest (CASSANDRA-3298) * Nodetool no longer leaks threads and closes JMX connections (CASSANDRA-3309) * (CQL) fix type information returned for wildcard queries (CASSANDRA-3311) + * fix truncate allowing data to be replayed post-restart (CASSANDRA-3297) 1.0.0-rc2 Propchange: cassandra/branches/cassandra-1.0.0/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009 /cassandra/branches/cassandra-0.7/contrib:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654 -/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689 Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689 Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Oct 5 18:33:38 2011 @@ -1,7 +1,7 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1170333,1172024 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654 -/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1178554,1178785 +/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689 Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original) +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Oct 5 18:33:38 2011 @@ -714,7 +714,7 @@ public class ColumnFamilyStore implement if (clean) { - logger.debug("forceFlush requested but everything is clean"); + logger.debug("forceFlush requested but everything is clean in {}", columnFamily); return null; } @@ -1593,7 +1593,7 @@ public class ColumnFamilyStore implement * @return a Future to the delete operation. Call the future's get() to make * sure the column family has been deleted */ - public Future<?> truncate() throws IOException + public Future<?> truncate() throws IOException, ExecutionException, InterruptedException { // We have two goals here: // - truncate should delete everything written before truncate was invoked @@ -1611,11 +1611,15 @@ public class ColumnFamilyStore implement // // Bonus bonus: simply forceFlush of all the CF is not enough, because if // for a given column family the memtable is clean, forceFlush will return - // immediately, even though there could be a memtable being flush at the same - // time. So to guarantee that all segments can be cleaned out, we need + // immediately, even though there could be a memtable being flushed at the same + // time. So to guarantee that all segments can be cleaned out, we need to // "waitForActiveFlushes" after the new segment has been created. + logger.debug("truncating {}", columnFamily); + // flush the CF being truncated before forcing the new segment + forceBlockingFlush(); CommitLog.instance.forceNewSegment(); ReplayPosition position = CommitLog.instance.getContext(); + // now flush everyone else. re-flushing ourselves is not necessary, but harmless for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) cfs.forceFlush(); waitForActiveFlushes(); Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java (original) +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java Wed Oct 5 18:33:38 2011 @@ -55,15 +55,6 @@ public class Truncation implements Messa this.columnFamily = columnFamily; } - /** - * This is equivalent to calling commit. Applies the changes to - * to the table that is obtained by calling Table.open(). - */ - public void apply() throws IOException - { - Table.open(keyspace).getColumnFamilyStore(columnFamily).truncate(); - } - public Message getMessage(Integer version) throws IOException { FastByteArrayOutputStream bos = new FastByteArrayOutputStream(); Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java (original) +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java Wed Oct 5 18:33:38 2011 @@ -131,6 +131,8 @@ public class CommitLog implements Commit public void resetUnsafe() { + for (CommitLogSegment segment : segments) + segment.close(); segments.clear(); segments.add(new CommitLogSegment()); } @@ -484,7 +486,6 @@ public class CommitLog implements Commit if (segment.isSafeToDelete() && iter.hasNext()) { logger.info("Discarding obsolete commit log:" + segment); - segment.close(); FileUtils.deleteAsync(segment.getPath()); // usually this will be the first (remaining) segment, but not always, if segment A contains // writes to a CF that is unflushed but is followed by segment B whose CFs are all flushed. @@ -532,34 +533,26 @@ public class CommitLog implements Commit return getSize(); } - public void forceNewSegment() + public void forceNewSegment() throws ExecutionException, InterruptedException { Callable<?> task = new Callable() { public Object call() throws IOException { - createNewSegment(); + if (currentSegment().length() > 0) + createNewSegment(); return null; } }; - try - { - executor.submit(task).get(); - } - catch (InterruptedException e) - { - throw new AssertionError(e); - } - catch (ExecutionException e) - { - throw new RuntimeException(e); - } + executor.submit(task).get(); } private void createNewSegment() throws IOException { + assert !segments.isEmpty(); sync(); + segments.getLast().close(); segments.add(new CommitLogSegment()); // Maintain desired CL size cap Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java (original) +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java Wed Oct 5 18:33:38 2011 @@ -48,6 +48,7 @@ public class CommitLogSegment public final long id; private final SequentialWriter logWriter; + private long finalSize = -1; // cache which cf is dirty in this segment to avoid having to lookup all ReplayPositions to decide if we could delete this segment public final Map<Integer, Integer> cfLastWrite = new HashMap<Integer, Integer>(); @@ -159,6 +160,9 @@ public class CommitLogSegment public long length() { + if (finalSize >= 0) + return finalSize; + try { return logWriter.length(); @@ -171,8 +175,12 @@ public class CommitLogSegment public void close() { + if (finalSize >= 0) + return; + try { + finalSize = logWriter.length(); logWriter.close(); } catch (IOException e) Modified: cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java?rev=1179370&r1=1179369&r2=1179370&view=diff ============================================================================== --- cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java (original) +++ cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java Wed Oct 5 18:33:38 2011 @@ -48,7 +48,7 @@ public class RecoveryManagerTruncateTest RowMutation rm; ColumnFamily cf; - // trucate clears memtable + // add a single cell rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("keymulti")); cf = ColumnFamily.create("Keyspace1", "Standard1"); cf.addColumn(column("col1", "val1", 1L)); @@ -60,22 +60,11 @@ public class RecoveryManagerTruncateTest // and now truncate it cfs.truncate().get(); + CommitLog.instance.resetUnsafe(); CommitLog.recover(); // and validate truncation. assertNull(getFromTable(table, "Standard1", "keymulti", "col1")); - - // truncate clears sstable - rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("keymulti")); - cf = ColumnFamily.create("Keyspace1", "Standard1"); - cf.addColumn(column("col1", "val1", 1L)); - rm.add(cf); - rm.apply(); - cfs.forceBlockingFlush(); - cfs.truncate().get(); - CommitLog.instance.resetUnsafe(); - CommitLog.recover(); - assertNull(getFromTable(table, "Standard1", "keymulti", "col1")); } private IColumn getFromTable(Table table, String cfName, String keyName, String columnName)