add getRangeKeySample and refactor key sampling to use more-efficient CFS.keySamples patch by Sam Tunnicliffe and jbellis for CASSANDRA-2917
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ba637b4d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ba637b4d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ba637b4d Branch: refs/heads/cassandra-1.0 Commit: ba637b4d1f1b0c48c62498cc35ac6f5665cf4f27 Parents: 1176d9f Author: Jonathan Ellis <jbel...@apache.org> Authored: Tue Feb 7 17:27:20 2012 -0600 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Wed Feb 8 17:53:47 2012 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/ColumnFamilyStore.java | 17 ++--- .../apache/cassandra/service/StorageService.java | 52 +++++++------- .../cassandra/service/StorageServiceMBean.java | 9 +++ src/java/org/apache/cassandra/tools/NodeCmd.java | 16 +++++ src/java/org/apache/cassandra/tools/NodeProbe.java | 6 ++ 6 files changed, 66 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index a37ec91..7018e57 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.1-dev + * add nodetool rangekeysample (CASSANDRA-2917) * Fix streaming too much data during move operations (CASSANDRA-3639) * Nodetool and CLI connect to localhost by default (CASSANDRA-3568) * Reduce memory used by primary index sample (CASSANDRA-3743) http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index e058e0d..b4d1602 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -29,7 +29,10 @@ import java.util.regex.Pattern; import javax.management.*; import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.service.CacheService; import org.slf4j.Logger; @@ -1493,16 +1496,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return Iterables.concat(stores); } - public Iterable<DecoratedKey<?>> allKeySamples() + public static List<ColumnFamilyStore> allUserDefined() { - Collection<SSTableReader> sstables = getSSTables(); - Iterable<DecoratedKey<?>>[] samples = new Iterable[sstables.size()]; - int i = 0; - for (SSTableReader sstable: sstables) - { - samples[i++] = sstable.getKeySamples(); - } - return Iterables.concat(samples); + List<ColumnFamilyStore> cfses = new ArrayList<ColumnFamilyStore>(); + for (Table table : Sets.difference(ImmutableSet.copyOf(Table.all()), ImmutableSet.of(Table.open(Table.SYSTEM_TABLE)))) + cfses.addAll(table.getColumnFamilyStores()); + return cfses; } public Iterable<DecoratedKey<?>> keySamples(Range<Token> range) http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 079bf0f..4c79373 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -44,8 +44,6 @@ import org.apache.cassandra.config.*; import org.apache.cassandra.db.*; import org.apache.cassandra.db.Table; import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.db.migration.AddKeyspace; -import org.apache.cassandra.db.migration.Migration; import org.apache.cassandra.dht.*; import org.apache.cassandra.gms.*; import org.apache.cassandra.io.sstable.SSTableDeletingTask; @@ -61,11 +59,7 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.ResponseVerbHandler; import org.apache.cassandra.service.AntiEntropyService.TreeRequestVerbHandler; import org.apache.cassandra.streaming.*; -import org.apache.cassandra.thrift.Constants; -import org.apache.cassandra.thrift.EndpointDetails; -import org.apache.cassandra.thrift.InvalidRequestException; -import org.apache.cassandra.thrift.TokenRange; -import org.apache.cassandra.thrift.UnavailableException; +import org.apache.cassandra.thrift.*; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.NodeId; import org.apache.cassandra.utils.Pair; @@ -2057,15 +2051,9 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe // we use the actual Range token for the first and last brackets of the splits to ensure correctness tokens.add(range.left); - List<DecoratedKey> keys = new ArrayList<DecoratedKey>(); Table t = Table.open(table); ColumnFamilyStore cfs = t.getColumnFamilyStore(cfName); - for (DecoratedKey sample : cfs.allKeySamples()) - { - if (range.contains(sample.token)) - keys.add(sample); - } - FBUtilities.sortSampledKeys(keys, range); + List<DecoratedKey> keys = keySamples(Collections.singleton(cfs), range); int splits = keys.size() * DatabaseDescriptor.getIndexInterval() / keysPerSplit; if (keys.size() >= splits) @@ -2081,22 +2069,21 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe return tokens; } + private List<DecoratedKey> keySamples(Iterable<ColumnFamilyStore> cfses, Range<Token> range) + { + List<DecoratedKey> keys = new ArrayList<DecoratedKey>(); + for (ColumnFamilyStore cfs : cfses) + Iterables.addAll(keys, cfs.keySamples(range)); + FBUtilities.sortSampledKeys(keys, range); + return keys; + } + /** return a token to which if a node bootstraps it will get about 1/2 of this node's range */ public Token getBootstrapToken() { Range<Token> range = getLocalPrimaryRange(); - List<DecoratedKey> keys = new ArrayList<DecoratedKey>(); - for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) - { - if (cfs.table.name.equals(Table.SYSTEM_TABLE)) - continue; - for (DecoratedKey key : cfs.allKeySamples()) - { - if (range.contains(key.token)) - keys.add(key); - } - } - FBUtilities.sortSampledKeys(keys, range); + + List<DecoratedKey> keys = keySamples(ColumnFamilyStore.allUserDefined(), range); Token token; if (keys.size() < 3) @@ -2904,4 +2891,17 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe { ColumnFamilyStore.loadNewSSTables(ksName, cfName); } + + /** + * #{@inheritDoc} + */ + public List<String> getRangeKeySample() + { + List<DecoratedKey> keys = keySamples(ColumnFamilyStore.allUserDefined(), getLocalPrimaryRange()); + + List<String> sampledKeys = new ArrayList<String>(); + for (DecoratedKey key : keys) + sampledKeys.add(key.getToken().toString()); + return sampledKeys; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/src/java/org/apache/cassandra/service/StorageServiceMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index c4c6f52..e8f0dd7 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -365,4 +365,13 @@ public interface StorageServiceMBean * @param cfName The ColumnFamily name where SSTables belong */ public void loadNewSSTables(String ksName, String cfName); + + /** + * Return a List of Tokens representing a sample of keys + * across all ColumnFamilyStores + * + * @return set of Tokens as Strings + */ + public List<String> getRangeKeySample(); + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/src/java/org/apache/cassandra/tools/NodeCmd.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index 870644b..25645c4 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -119,6 +119,7 @@ public class NodeCmd UPGRADESSTABLES, VERSION, DESCRIBERING, + RANGEKEYSAMPLE, } @@ -156,6 +157,7 @@ public class NodeCmd addCmdHelp(header, "setcompactionthroughput <value_in_mb>", "Set the MB/s throughput cap for compaction in the system, or 0 to disable throttling."); addCmdHelp(header, "setstreamthroughput <value_in_mb>", "Set the MB/s throughput cap for streaming in the system, or 0 to disable throttling."); addCmdHelp(header, "describering [keyspace]", "Shows the token ranges info of a given keyspace."); + addCmdHelp(header, "rangekeysample", "Shows the sampled keys held across all keyspaces."); addCmdHelp(header, "rebuild [src-dc-name]", "Rebuild data by streaming from other nodes (similarly to bootstrap)"); // Two args @@ -781,6 +783,10 @@ public class NodeCmd nodeCmd.printDescribeRing(arguments[0], System.out); break; + case RANGEKEYSAMPLE : + nodeCmd.printRangeKeySample(System.out); + break; + default : throw new RuntimeException("Unreachable code."); } @@ -818,6 +824,16 @@ public class NodeCmd } } + private void printRangeKeySample(PrintStream outs) + { + outs.println("RangeKeySample: "); + List<String> tokenStrings = this.probe.getRangeKeySample(); + for (String tokenString : tokenStrings) + { + outs.println("\t" + tokenString); + } + } + private void printGossipInfo(PrintStream out) { out.println(probe.getGossipInfo()); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ba637b4d/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 22678ae..469c2a5 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -643,6 +643,12 @@ public class NodeProbe { ssProxy.rebuild(sourceDc); } + + public List<String> getRangeKeySample() + { + return ssProxy.getRangeKeySample(); + } + } class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>