Author: jbellis Date: Fri Jan 28 16:27:44 2011 New Revision: 1064764 URL: http://svn.apache.org/viewvc?rev=1064764&view=rev Log: merge from 0.7
Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java Propchange: cassandra/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7:1026516-1064342 +/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7:1026516-1064763 /cassandra/branches/cassandra-0.7.0:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3:774578-796573 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Fri Jan 28 16:27:44 2011 @@ -16,6 +16,8 @@ * fix possible ByteBuffer race conditions (CASSANDRA-2066) * reduce garbage generated by MessagingService to prevent load spikes (CASSANDRA-2058) + * fix math in RandomPartitioner.describeOwnership (CASSANDRA-2071) + * fix deletion of sstable non-data components (CASSANDRA-2059) 0.7.1 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1064342 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1064763 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1064342 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1064763 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1064342 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1064763 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1064342 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1064763 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Jan 28 16:27:44 2011 @@ -1,5 +1,5 @@ -/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1055311,1056121,1057932,1064193 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1064342 +/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1064713 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1064763 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198 Modified: cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java Fri Jan 28 16:27:44 2011 @@ -197,40 +197,41 @@ public class HintedHandOffManager // 3. Delete the subcolumn if the write was successful // 4. Force a flush // 5. Do major compaction to clean up all deletes etc. - DecoratedKey epkey = StorageService.getPartitioner().decorateKey(ByteBuffer.wrap(endpoint.getHostAddress().getBytes(UTF_8))); + ByteBuffer endpointAsUTF8 = ByteBuffer.wrap(endpoint.getHostAddress().getBytes(UTF_8)); // keys have to be UTF8 to make OPP happy + DecoratedKey epkey = StorageService.getPartitioner().decorateKey(endpointAsUTF8); int rowsReplayed = 0; ColumnFamilyStore hintStore = Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(HINTS_CF); ByteBuffer startColumn = ByteBufferUtil.EMPTY_BYTE_BUFFER; + delivery: - while (true) + while (true) + { + QueryFilter filter = QueryFilter.getSliceFilter(epkey, new QueryPath(HINTS_CF), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, PAGE_SIZE); + ColumnFamily hintColumnFamily = ColumnFamilyStore.removeDeleted(hintStore.getColumnFamily(filter), Integer.MAX_VALUE); + if (pagingFinished(hintColumnFamily, startColumn)) + break; + for (IColumn keyColumn : hintColumnFamily.getSortedColumns()) { - QueryFilter filter = QueryFilter.getSliceFilter(epkey, new QueryPath(HINTS_CF), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, PAGE_SIZE); - ColumnFamily hintColumnFamily = ColumnFamilyStore.removeDeleted(hintStore.getColumnFamily(filter), Integer.MAX_VALUE); - if (pagingFinished(hintColumnFamily, startColumn)) - break; - Collection<IColumn> keyColumns = hintColumnFamily.getSortedColumns(); - for (IColumn keyColumn : keyColumns) + startColumn = keyColumn.name(); + Collection<IColumn> tableCFs = keyColumn.getSubColumns(); + for (IColumn tableCF : tableCFs) { - startColumn = keyColumn.name(); - Collection<IColumn> tableCFs = keyColumn.getSubColumns(); - for (IColumn tableCF : tableCFs) + String[] parts = getTableAndCFNames(tableCF.name()); + if (sendMessage(endpoint, parts[0], parts[1], keyColumn.name())) { - String[] parts = getTableAndCFNames(tableCF.name()); - if (sendMessage(endpoint, parts[0], parts[1], keyColumn.name())) - { - deleteHintKey(ByteBuffer.wrap(endpoint.getHostAddress().getBytes(UTF_8)), keyColumn.name(), tableCF.name(), tableCF.timestamp()); - rowsReplayed++; - } - else - { - logger_.info("Could not complete hinted handoff to " + endpoint); - break delivery; - } - - startColumn = keyColumn.name(); + deleteHintKey(endpointAsUTF8, keyColumn.name(), tableCF.name(), tableCF.timestamp()); + rowsReplayed++; } + else + { + logger_.info("Could not complete hinted handoff to " + endpoint); + break delivery; + } + + startColumn = keyColumn.name(); } } + } if (rowsReplayed > 0) { Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java Fri Jan 28 16:27:44 2011 @@ -148,15 +148,14 @@ public class RandomPartitioner implement Token start = (Token)i.next(); BigInteger ti = ((BigIntegerToken)start).token; // The first token and its value Token t; BigInteger tim1 = ti; // The last token and its value (after loop) while (i.hasNext()) { - t = (Token)i.next(); ti = ((BigIntegerToken)t).token; // The next token and its value - float x = new BigDecimal(ti.subtract(tim1)).divide(r).floatValue(); // %age = T(i) - T(i-1) / R - ownerships.put(t, x); // save (T(i) -> %age) - tim1 = ti; // -> advance loop + t = (Token)i.next(); ti = ((BigIntegerToken)t).token; // The next token and its value + float x = new BigDecimal(ti.subtract(tim1).add(ri).mod(ri)).divide(r).floatValue(); // %age = ((T(i) - T(i-1) + R) % R) / R + ownerships.put(t, x); // save (T(i) -> %age) + tim1 = ti; // -> advance loop } - // The start token's range extends backward to the last token, which is why both were saved - // above. The simple calculation for this is: T(start) - T(end) + r % r / r. - // (In the 1-case, this produces 0% instead of 100%.) - ownerships.put(start, new BigDecimal(((BigIntegerToken)start).token.subtract(ti).add(ri).mod(ri)).divide(r).floatValue()); + // The start token's range extends backward to the last token, which is why both were saved above. + float x = new BigDecimal(((BigIntegerToken)start).token.subtract(ti).add(ri).mod(ri)).divide(r).floatValue(); + ownerships.put(start, x); } return ownerships; } Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java Fri Jan 28 16:27:44 2011 @@ -23,9 +23,11 @@ import java.io.File; import java.io.IOException; import java.lang.ref.PhantomReference; import java.lang.ref.ReferenceQueue; +import java.util.Collections; import java.util.Set; import java.util.concurrent.TimeUnit; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,8 +93,7 @@ public class SSTableDeletingReference ex } } // let the remainder be cleaned up by delete - components.remove(Component.DATA); - SSTable.delete(desc, components); + SSTable.delete(desc, Sets.difference(components, Collections.singleton(Component.DATA))); tracker.spaceReclaimed(size); } } Modified: cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java Fri Jan 28 16:27:44 2011 @@ -20,6 +20,7 @@ package org.apache.cassandra.locator; import java.lang.management.ManagementFactory; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingDeque; @@ -252,6 +253,20 @@ public class DynamicEndpointSnitch exten return subsnitch.getClass().getName(); } + public List<Double> dumpTimings(String hostname) throws UnknownHostException + { + InetAddress host = InetAddress.getByName(hostname); + ArrayList<Double> timings = new ArrayList<Double>(); + AdaptiveLatencyTracker window = windows.get(host); + if (window != null) + { + for (double time: window) + { + timings.add(time); + } + } + return timings; + } } @@ -307,4 +322,5 @@ class AdaptiveLatencyTracker extends Abs } return log; } + } Modified: cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java?rev=1064764&r1=1064763&r2=1064764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/locator/DynamicEndpointSnitchMBean.java Fri Jan 28 16:27:44 2011 @@ -20,7 +20,9 @@ package org.apache.cassandra.locator; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Map; +import java.util.List; public interface DynamicEndpointSnitchMBean { public Map<InetAddress, Double> getScores(); @@ -28,4 +30,5 @@ public interface DynamicEndpointSnitchMB public int getResetInterval(); public double getBadnessThreshold(); public String getSubsnitchClassName(); + public List<Double> dumpTimings(String hostname) throws UnknownHostException; }