Repository: cassandra Updated Branches: refs/heads/trunk f61cc0d88 -> a808769fc
Group pending compactions based on table Patch by Taiyuan Zhang; reviewed by marcuse for CASSANDRA-10718 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a808769f Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a808769f Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a808769f Branch: refs/heads/trunk Commit: a808769fc6cc73162109453f30202d86936e1492 Parents: f61cc0d Author: Taiyuan Zhang <taiyu...@andrew.cmu.edu> Authored: Fri Dec 11 15:16:10 2015 +0100 Committer: Marcus Eriksson <marc...@apache.org> Committed: Mon Dec 14 08:38:53 2015 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/metrics/CompactionMetrics.java | 52 ++++++++++++++++++++ .../org/apache/cassandra/tools/NodeProbe.java | 1 + .../tools/nodetool/CompactionStats.java | 30 ++++++++++- 4 files changed, 83 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a808769f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 905f7a3..10b6226 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.2 + * Group pending compactions based on table (CASSANDRA-10718) * Add compressor name in sstablemetadata output (CASSANDRA-9879) * Fix type casting for counter columns (CASSANDRA-10824) * Prevent running Cassandra as root (CASSANDRA-8142) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a808769f/src/java/org/apache/cassandra/metrics/CompactionMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java index 19eadc8..f3a9d00 100644 --- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java +++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java @@ -24,6 +24,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Meter; +import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -44,6 +45,9 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt /** Estimated number of compactions remaining to perform */ public final Gauge<Integer> pendingTasks; + /** Estimated number of compactions remaining to perform, group by keyspace and then table name */ + public final Gauge<Map<String, Map<String, Integer>>> pendingTasksByTableName; + /** Number of completed compactions since server [re]start */ public final Gauge<Long> completedTasks; /** Total number of compactions since server [re]start */ @@ -68,6 +72,54 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt return n + compactions.size(); } }); + + pendingTasksByTableName = Metrics.register(factory.createMetricName("PendingTasksByTableName"), + new Gauge<Map<String, Map<String, Integer>>>() + { + @Override + public Map<String, Map<String, Integer>> getValue() { + Map<String, Map<String, Integer>> resultMap = new HashMap<>(); + // estimation of compactions need to be done + for (String keyspaceName : Schema.instance.getKeyspaces()) + { + for (ColumnFamilyStore cfs : Keyspace.open(keyspaceName).getColumnFamilyStores()) + { + int taskNumber = cfs.getCompactionStrategyManager().getEstimatedRemainingTasks(); + if (taskNumber > 0) + { + if (!resultMap.containsKey(keyspaceName)) + { + resultMap.put(keyspaceName, new HashMap<>()); + } + resultMap.get(keyspaceName).put(cfs.getTableName(), taskNumber); + } + } + } + + // currently running compactions + for (CompactionInfo.Holder compaction : compactions) + { + CFMetaData metaData = compaction.getCompactionInfo().getCFMetaData(); + if (!resultMap.containsKey(metaData.ksName)) + { + resultMap.put(metaData.ksName, new HashMap<>()); + } + + Map<String, Integer> tableNameToCountMap = resultMap.get(metaData.ksName); + if (tableNameToCountMap.containsKey(metaData.cfName)) + { + tableNameToCountMap.put(metaData.cfName, + tableNameToCountMap.get(metaData.cfName) + 1); + } + else + { + tableNameToCountMap.put(metaData.cfName, 1); + } + } + return resultMap; + } + }); + completedTasks = Metrics.register(factory.createMetricName("CompletedTasks"), new Gauge<Long>() { public Long getValue() http://git-wip-us.apache.org/repos/asf/cassandra/blob/a808769f/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 099876b..1078004 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -1195,6 +1195,7 @@ public class NodeProbe implements AutoCloseable CassandraMetricsRegistry.JmxCounterMBean.class); case "CompletedTasks": case "PendingTasks": + case "PendingTasksByTableName": return JMX.newMBeanProxy(mbeanServerConn, new ObjectName("org.apache.cassandra.metrics:type=Compaction,name=" + metricName), CassandraMetricsRegistry.JmxGaugeMBean.class).getValue(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/a808769f/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java index e57d2ee..b950a1c 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java +++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java @@ -25,6 +25,7 @@ import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.cassandra.db.compaction.CompactionManagerMBean; import org.apache.cassandra.db.compaction.OperationType; @@ -45,7 +46,34 @@ public class CompactionStats extends NodeToolCmd public void execute(NodeProbe probe) { CompactionManagerMBean cm = probe.getCompactionManagerProxy(); - System.out.println("pending tasks: " + probe.getCompactionMetric("PendingTasks")); + Map<String, Map<String, Integer>> pendingTaskNumberByTable = + (Map<String, Map<String, Integer>>) probe.getCompactionMetric("PendingTasksByTableName"); + int numTotalPendingTask = 0; + for (Entry<String, Map<String, Integer>> ksEntry : pendingTaskNumberByTable.entrySet()) + { + for (Entry<String, Integer> tableEntry : ksEntry.getValue().entrySet()) + numTotalPendingTask += tableEntry.getValue(); + } + System.out.println("pending tasks: "+numTotalPendingTask); + for (Entry<String, Map<String, Integer>> ksEntry : pendingTaskNumberByTable.entrySet()) + { + String ksName = ksEntry.getKey(); + for (Entry<String, Integer> tableEntry : ksEntry.getValue().entrySet()) + { + String tableName = tableEntry.getKey(); + int pendingTaskCount = tableEntry.getValue(); + + StringBuilder builder = new StringBuilder(); + builder.append("- "); + builder.append(ksName); + builder.append('.'); + builder.append(tableName); + builder.append(": "); + builder.append(pendingTaskCount); + System.out.println(builder.toString()); + } + } + System.out.println(); long remainingBytes = 0; List<Map<String, String>> compactions = cm.getCompactions(); if (!compactions.isEmpty())
