Merge branch 'cassandra-2.1' into trunk Conflicts: src/java/org/apache/cassandra/tools/NodeTool.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0a09b87d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0a09b87d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0a09b87d Branch: refs/heads/trunk Commit: 0a09b87dcefee496dbfa6a398fb18033e32042d5 Parents: 184bb65 faf9181 Author: Brandon Williams <brandonwilli...@apache.org> Authored: Wed Jan 21 11:49:08 2015 -0600 Committer: Brandon Williams <brandonwilli...@apache.org> Committed: Wed Jan 21 11:49:08 2015 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 69 ++++++++- .../cassandra/db/ColumnFamilyStoreMBean.java | 14 ++ .../cassandra/metrics/ColumnFamilyMetrics.java | 19 ++- .../org/apache/cassandra/tools/NodeProbe.java | 26 +++- .../org/apache/cassandra/tools/NodeTool.java | 85 ++++++++++- .../org/apache/cassandra/utils/TopKSampler.java | 139 ++++++++++++++++++ .../apache/cassandra/utils/TopKSamplerTest.java | 147 +++++++++++++++++++ 8 files changed, 483 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 66b17ce,f1eaa77..9343d66 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,58 -1,5 +1,59 @@@ +3.0 + * Make CassandraException unchecked, extend RuntimeException (CASSANDRA-8560) + * Support direct buffer decompression for reads (CASSANDRA-8464) + * DirectByteBuffer compatible LZ4 methods (CASSANDRA-7039) + * Add role based access control (CASSANDRA-7653) + * Group sstables for anticompaction correctly (CASSANDRA-8578) + * Add ReadFailureException to native protocol, respond + immediately when replicas encounter errors while handling + a read request (CASSANDRA-7886) + * Switch CommitLogSegment from RandomAccessFile to nio (CASSANDRA-8308) + * Allow mixing token and partition key restrictions (CASSANDRA-7016) + * Support index key/value entries on map collections (CASSANDRA-8473) + * Modernize schema tables (CASSANDRA-8261) + * Support for user-defined aggregation functions (CASSANDRA-8053) + * Fix NPE in SelectStatement with empty IN values (CASSANDRA-8419) + * Refactor SelectStatement, return IN results in natural order instead + of IN value list order (CASSANDRA-7981) + * Support UDTs, tuples, and collections in user-defined + functions (CASSANDRA-7563) + * Fix aggregate fn results on empty selection, result column name, + and cqlsh parsing (CASSANDRA-8229) + * Mark sstables as repaired after full repair (CASSANDRA-7586) + * Extend Descriptor to include a format value and refactor reader/writer + APIs (CASSANDRA-7443) + * Integrate JMH for microbenchmarks (CASSANDRA-8151) + * Keep sstable levels when bootstrapping (CASSANDRA-7460) + * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838) + * Support for aggregation functions (CASSANDRA-4914) + * Remove cassandra-cli (CASSANDRA-7920) + * Accept dollar quoted strings in CQL (CASSANDRA-7769) + * Make assassinate a first class command (CASSANDRA-7935) + * Support IN clause on any clustering column (CASSANDRA-4762) + * Improve compaction logging (CASSANDRA-7818) + * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917) + * Do anticompaction in groups (CASSANDRA-6851) + * Support user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929, + 7924, 7812, 8063, 7813, 7708) + * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416) + * Move sstable RandomAccessReader to nio2, which allows using the + FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050) + * Remove CQL2 (CASSANDRA-5918) + * Add Thrift get_multi_slice call (CASSANDRA-6757) + * Optimize fetching multiple cells by name (CASSANDRA-6933) + * Allow compilation in java 8 (CASSANDRA-7028) + * Make incremental repair default (CASSANDRA-7250) + * Enable code coverage thru JaCoCo (CASSANDRA-7226) + * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369) + * Shorten SSTable path (CASSANDRA-6962) + * Use unsafe mutations for most unit tests (CASSANDRA-6969) + * Fix race condition during calculation of pending ranges (CASSANDRA-7390) + * Fail on very large batch sizes (CASSANDRA-8011) + * Improve concurrency of repair (CASSANDRA-6455, 8208) + + 2.1.3 + * Add tooling to detect hot partitions (CASSANDRA-7974) * Fix cassandra-stress user-mode truncation of partition generation (CASSANDRA-8608) * Only stream from unrepaired sstables during inc repair (CASSANDRA-8267) * Don't allow starting multiple inc repairs on the same sstables (CASSANDRA-8316) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java index d763dea,c82569d..07246cf --- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java @@@ -25,9 -24,10 +24,10 @@@ import java.util.concurrent.TimeUnit import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.io.sstable.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.utils.EstimatedHistogram; + import org.apache.cassandra.utils.TopKSampler; import com.google.common.collect.Maps; import com.google.common.collect.Sets; http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/tools/NodeProbe.java index ea24530,67cc7f1..a6ab091 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@@ -37,13 -36,12 +36,13 @@@ import javax.management.remote.JMXConne import javax.management.remote.JMXConnector; import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; - import javax.management.openmbean.TabularData; + import javax.management.openmbean.*; import com.google.common.base.Function; - import com.google.common.collect.Iterables; - import com.google.common.collect.Sets; + import com.google.common.collect.*; + import com.google.common.util.concurrent.Uninterruptibles; +import com.google.common.util.concurrent.AbstractFuture; import com.yammer.metrics.reporting.JmxReporter; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutorMBean; import org.apache.cassandra.db.ColumnFamilyStoreMBean; @@@ -53,9 -51,8 +52,10 @@@ import org.apache.cassandra.db.compacti import org.apache.cassandra.db.compaction.CompactionManagerMBean; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.FailureDetectorMBean; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.GossiperMBean; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; + import org.apache.cassandra.metrics.ColumnFamilyMetrics.Sampler; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingServiceMBean; import org.apache.cassandra.repair.RepairParallelism; http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a09b87d/src/java/org/apache/cassandra/tools/NodeTool.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/tools/NodeTool.java index 4a5e352,12496fc..24772d7 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@@ -46,8 -44,9 +44,9 @@@ import org.apache.cassandra.db.compacti import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; -import org.apache.cassandra.locator.LocalStrategy; + import org.apache.cassandra.metrics.ColumnFamilyMetrics.Sampler; import org.apache.cassandra.net.MessagingServiceMBean; +import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.service.CacheServiceMBean; import org.apache.cassandra.streaming.ProgressInfo; @@@ -930,10 -925,85 +930,85 @@@ public class NodeToo } } - @Command(name = "cfhistograms", description = "Print statistic histograms for a given table") + @Command(name = "toppartitions", description = "Sample and print the most active partitions for a given column family") + public static class TopPartitions extends NodeToolCmd + { + @Arguments(usage = "<keyspace> <cfname> <duration>", description = "The keyspace, column family name, and duration in milliseconds") + private List<String> args = new ArrayList<>(); + @Option(name = "-s", description = "Capacity of stream summary, closer to the actual cardinality of partitions will yield more accurate results (Default: 256)") + private int size = 256; + @Option(name = "-k", description = "Number of the top partitions to list (Default: 10)") + private int topCount = 10; + @Option(name = "-a", description = "Comma separated list of samplers to use (Default: all)") + private String samplers = join(Sampler.values(), ','); + @Override + public void execute(NodeProbe probe) + { + checkArgument(args.size() == 3, "toppartitions requires keyspace, column family name, and duration"); + checkArgument(topCount < size, "TopK count (-k) option must be smaller then the summary capacity (-s)"); + String keyspace = args.get(0); + String cfname = args.get(1); + Integer duration = Integer.parseInt(args.get(2)); + // generate the list of samplers + List<Sampler> targets = Lists.newArrayList(); + for (String s : samplers.split(",")) + { + try + { + targets.add(Sampler.valueOf(s.toUpperCase())); + } catch (Exception e) + { + throw new IllegalArgumentException(s + " is not a valid sampler, choose one of: " + join(Sampler.values(), ", ")); + } + } + + Map<Sampler, CompositeData> results; + try + { + results = probe.getPartitionSample(keyspace, cfname, size, duration, topCount, targets); + } catch (OpenDataException e) + { + throw new RuntimeException(e); + } + boolean first = true; + for(Entry<Sampler, CompositeData> result : results.entrySet()) + { + CompositeData sampling = result.getValue(); + // weird casting for http://bugs.sun.com/view_bug.do?bug_id=6548436 + List<CompositeData> topk = (List<CompositeData>) (Object) Lists.newArrayList(((TabularDataSupport) sampling.get("partitions")).values()); + Collections.sort(topk, new Ordering<CompositeData>() + { + public int compare(CompositeData left, CompositeData right) + { + return Long.compare((long) right.get("count"), (long) left.get("count")); + } + }); + if(!first) + System.out.println(); + System.out.println(result.getKey().toString()+ " Sampler:"); + System.out.printf(" Cardinality: ~%d (%d capacity)%n", (long) sampling.get("cardinality"), size); + System.out.printf(" Top %d partitions:%n", topCount); + if (topk.size() == 0) + { + System.out.println("\tNothing recorded during sampling period..."); + } else + { + int offset = 0; + for (CompositeData entry : topk) + offset = Math.max(offset, entry.get("string").toString().length()); + System.out.printf("\t%-" + offset + "s%10s%10s%n", "Partition", "Count", "+/-"); + for (CompositeData entry : topk) + System.out.printf("\t%-" + offset + "s%10d%10d%n", entry.get("string").toString(), entry.get("count"), entry.get("error")); + } + first = false; + } + } + } + + @Command(name = "cfhistograms", description = "Print statistic histograms for a given column family") public static class CfHistograms extends NodeToolCmd { - @Arguments(usage = "<keyspace> <cfname>", description = "The keyspace and column family name") + @Arguments(usage = "<keyspace> <table>", description = "The keyspace and table name") private List<String> args = new ArrayList<>(); @Override