Author: slebresne Date: Fri Nov 4 08:53:17 2011 New Revision: 1197431 URL: http://svn.apache.org/viewvc?rev=1197431&view=rev Log: merge from 1.0
Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/contrib/ (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/db/ReadCommand.java cassandra/trunk/src/java/org/apache/cassandra/db/SliceFromReadCommand.java cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java Propchange: cassandra/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0:1125021-1130369 /cassandra/branches/cassandra-0.8.1:1101014-1125018 -/cassandra/branches/cassandra-1.0:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1197431&r1=1197430&r2=1197431&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Fri Nov 4 08:53:17 2011 @@ -17,6 +17,7 @@ * add JMX call to clean (failed) repair sessions (CASSANDRA-3316) * fix sstableloader reference acquisition bug (CASSANDRA-3438) * fix estimated row size regression (CASSANDRA-3451) + * make sure we don't return more columns than asked (CASSANDRA-3303, 3395) Merged from 0.8: * acquire compactionlock during truncate (CASSANDRA-3399) * fix displaying cfdef entries for super columnfamilies (CASSANDRA-3415) Propchange: cassandra/trunk/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018 -/cassandra/branches/cassandra-1.0/contrib:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/contrib:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Nov 4 08:53:17 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1196956 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1196958,1197123,1197130,1197417,1197420 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1196958,1197123,1197130,1197417,1197420,1197426 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020 Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ReadCommand.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ReadCommand.java?rev=1197431&r1=1197430&r2=1197431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/ReadCommand.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/ReadCommand.java Fri Nov 4 08:53:17 2011 @@ -31,6 +31,7 @@ import org.apache.cassandra.io.IVersione import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageProducer; import org.apache.cassandra.service.IReadCommand; +import org.apache.cassandra.service.RepairCallback; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; @@ -66,7 +67,7 @@ public abstract class ReadCommand implem this.queryPath = queryPath; this.commandType = cmdType; } - + public boolean isDigestQuery() { return isDigestQuery; @@ -81,7 +82,7 @@ public abstract class ReadCommand implem { return queryPath.columnFamilyName; } - + public abstract ReadCommand copy(); public abstract Row getRow(Table table) throws IOException; @@ -95,6 +96,18 @@ public abstract class ReadCommand implem { return table; } + + // maybeGenerateRetryCommand is used to generate a retry for short reads + public ReadCommand maybeGenerateRetryCommand(RepairCallback handler, Row row) + { + return null; + } + + // maybeTrim removes columns from a response that is too long + public void maybeTrim(Row row) + { + // noop + } } class ReadCommandSerializer implements IVersionedSerializer<ReadCommand> Modified: cassandra/trunk/src/java/org/apache/cassandra/db/SliceFromReadCommand.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/SliceFromReadCommand.java?rev=1197431&r1=1197430&r2=1197431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/SliceFromReadCommand.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/SliceFromReadCommand.java Fri Nov 4 08:53:17 2011 @@ -19,17 +19,25 @@ package org.apache.cassandra.db; import java.io.*; import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.service.RepairCallback; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.thrift.ColumnParent; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SliceFromReadCommand extends ReadCommand { + static final Logger logger = LoggerFactory.getLogger(SliceFromReadCommand.class); + public final ByteBuffer start, finish; public final boolean reversed; public final int count; @@ -62,6 +70,64 @@ public class SliceFromReadCommand extend } @Override + public ReadCommand maybeGenerateRetryCommand(RepairCallback handler, Row row) + { + int maxLiveColumns = handler.getMaxLiveColumns(); + int liveColumnsInRow = row != null ? row.cf.getLiveColumnCount() : 0; + + assert maxLiveColumns <= count; + if ((maxLiveColumns == count) && (liveColumnsInRow < count)) + { + int retryCount = count + count - liveColumnsInRow; + return new RetriedSliceFromReadCommand(table, key, queryPath, start, finish, reversed, count, retryCount); + } + + return null; + } + + @Override + public void maybeTrim(Row row) + { + if ((row == null) || (row.cf == null)) + return; + + int liveColumnsInRow = row.cf.getLiveColumnCount(); + + if (liveColumnsInRow > getRequestedCount()) + { + int columnsToTrim = liveColumnsInRow - getRequestedCount(); + + logger.debug("trimming {} live columns to the originally requested {}", row.cf.getLiveColumnCount(), getRequestedCount()); + + Collection<IColumn> columns; + if (reversed) + columns = row.cf.getSortedColumns(); + else + columns = row.cf.getReverseSortedColumns(); + + Collection<ByteBuffer> toRemove = new HashSet<ByteBuffer>(); + + Iterator<IColumn> columnIterator = columns.iterator(); + while (columnIterator.hasNext() && (toRemove.size() < columnsToTrim)) + { + IColumn column = columnIterator.next(); + if (column.isLive()) + toRemove.add(column.name()); + } + + for (ByteBuffer columnName : toRemove) + { + row.cf.remove(columnName); + } + } + } + + protected int getRequestedCount() + { + return count; + } + + @Override public String toString() { return "SliceFromReadCommand(" + Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java?rev=1197431&r1=1197430&r2=1197431&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java Fri Nov 4 08:53:17 2011 @@ -685,7 +685,10 @@ public class StorageProxy implements Sto long startTime2 = System.currentTimeMillis(); Row row = handler.get(); if (row != null) + { + command.maybeTrim(row); rows.add(row); + } if (logger.isDebugEnabled()) logger.debug("Read: " + (System.currentTimeMillis() - startTime2) + " ms."); @@ -739,35 +742,21 @@ public class StorageProxy implements Sto throw new AssertionError(e); // full data requested from each node here, no digests should be sent } - // retry short reads, otherwise add the row to our resultset - if (command instanceof SliceFromReadCommand) + ReadCommand retryCommand = command.maybeGenerateRetryCommand(handler, row); + if (retryCommand != null) { - // short reads are only possible on SliceFromReadCommand - SliceFromReadCommand sliceCommand = (SliceFromReadCommand) command; - int maxLiveColumns = handler.getMaxLiveColumns(); - int liveColumnsInRow = row != null ? row.cf.getLiveColumnCount() : 0; - - assert maxLiveColumns <= sliceCommand.count; - if ((maxLiveColumns == sliceCommand.count) && (liveColumnsInRow < sliceCommand.count)) - { - logger.debug("detected short read: expected {} columns, but only resolved {} columns", - sliceCommand.count, liveColumnsInRow); + logger.debug("issuing retry for read command"); + if (commandsToRetry == Collections.EMPTY_LIST) + commandsToRetry = new ArrayList<ReadCommand>(); + commandsToRetry.add(retryCommand); + continue; + } - int retryCount = sliceCommand.count + sliceCommand.count - liveColumnsInRow; - SliceFromReadCommand retryCommand = new SliceFromReadCommand(command.table, - command.key, - command.queryPath, - sliceCommand.start, - sliceCommand.finish, - sliceCommand.reversed, - retryCount); - if (commandsToRetry == Collections.EMPTY_LIST) - commandsToRetry = new ArrayList<ReadCommand>(); - commandsToRetry.add(retryCommand); - continue; - } + if (row != null) + { + command.maybeTrim(row); + rows.add(row); } - rows.add(row); } } } while (!commandsToRetry.isEmpty());