This is an automated email from the ASF dual-hosted git repository. jmckenzie pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 2ce9b13 Add logging and metrics for index summary redistribution 2ce9b13 is described below commit 2ce9b13e8ff1d5d54d87e403a25561a82c9490a0 Author: Marcus Eriksson <marc...@apache.org> AuthorDate: Tue Oct 12 12:28:35 2021 -0400 Add logging and metrics for index summary redistribution Patch by Marcus Eriksson, reviewed by David Capwell for CASSANDRA-17036 Co-authored-by: Marcus Eriksson <marc...@apache.org> Co-authored-by: Josh McKenzie <jmcken...@apache.org> --- CHANGES.txt | 1 + .../org/apache/cassandra/io/sstable/IndexSummaryManager.java | 4 +++- .../cassandra/io/sstable/IndexSummaryRedistribution.java | 12 ++++++++---- .../apache/cassandra/io/sstable/format/SSTableReader.java | 2 +- src/java/org/apache/cassandra/metrics/CompactionMetrics.java | 5 ++++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 384576b..0c3dd81 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * Add metrics and logging around index summary redistribution (CASSANDRA-17036) * Add configuration options for minimum allowable replication factor and default replication factor (CASSANDRA-14557) * Expose information about stored hints via a nodetool command and a virtual table (CASSANDRA-14795) * Add broadcast_rpc_address to system.local (CASSANDRA-11181) diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java index aac0be4..ee6f349 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java @@ -30,6 +30,8 @@ import java.util.concurrent.TimeoutException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; + +import com.codahale.metrics.Timer; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -231,7 +233,7 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean Pair<Long, Map<TableId, LifecycleTransaction>> redistributionTransactionInfo = getRestributionTransactions(); Map<TableId, LifecycleTransaction> transactions = redistributionTransactionInfo.right; long nonRedistributingOffHeapSize = redistributionTransactionInfo.left; - try + try (Timer.Context ctx = CompactionManager.instance.getMetrics().indexSummaryRedistributionTime.time()) { redistributeSummaries(new IndexSummaryRedistribution(transactions, nonRedistributingOffHeapSize, diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java index 90a8621..bac3280 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; @@ -46,6 +47,7 @@ import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; public class IndexSummaryRedistribution extends CompactionInfo.Holder { @@ -80,6 +82,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder public List<SSTableReader> redistributeSummaries() throws IOException { + long start = nanoTime(); logger.info("Redistributing index summaries"); List<SSTableReader> redistribute = new ArrayList<>(); for (LifecycleTransaction txn : transactions.values()) @@ -91,7 +94,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder for (SSTableReader sstable : redistribute) total += sstable.getIndexSummaryOffHeapSize(); - logger.trace("Beginning redistribution of index summaries for {} sstables with memory pool size {} MB; current spaced used is {} MB", + logger.info("Beginning redistribution of index summaries for {} sstables with memory pool size {} MB; current spaced used is {} MB", redistribute.size(), memoryPoolBytes / 1024L / 1024L, total / 1024.0 / 1024.0); final Map<SSTableReader, Double> readRates = new HashMap<>(redistribute.size()); @@ -129,9 +132,9 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder total = nonRedistributingOffHeapSize; for (SSTableReader sstable : newSSTables) total += sstable.getIndexSummaryOffHeapSize(); - if (logger.isTraceEnabled()) - logger.trace("Completed resizing of index summaries; current approximate memory used: {}", - FBUtilities.prettyPrintMemory(total)); + + logger.info("Completed resizing of index summaries; current approximate memory used: {} MB, time spent: {}ms", + total / 1024.0 / 1024.0, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start)); return newSSTables; } @@ -243,6 +246,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder } // downsample first, then upsample + logger.info("index summaries: downsample: {}, force resample: {}, upsample: {}, force upsample: {}", toDownsample.size(), forceResample.size(), toUpsample.size(), forceUpsample.size()); toDownsample.addAll(forceResample); toDownsample.addAll(toUpsample); toDownsample.addAll(forceUpsample); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 4af5743..771974d 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -742,7 +742,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS } catch (IOException e) { - logger.trace("Cannot save SSTable Summary: ", e); + logger.error("Cannot save SSTable Summary: ", e); // corrupted hence delete it and let it load it now. if (summariesFile.exists()) diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java index 375adc4..0fe1ec7 100644 --- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java +++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java @@ -23,6 +23,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Meter; +import com.codahale.metrics.Timer; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -51,7 +52,8 @@ public class CompactionMetrics public final Meter totalCompactionsCompleted; /** Total number of bytes compacted since server [re]start */ public final Counter bytesCompacted; - + /** Time spent redistributing index summaries */ + public final Timer indexSummaryRedistributionTime; /** Total number of compactions that have had sstables drop out of them */ public final Counter compactionsReduced; @@ -149,5 +151,6 @@ public class CompactionMetrics compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced")); sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction")); compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted")); + indexSummaryRedistributionTime = Metrics.timer(factory.createMetricName("IndexSummaryRedistributionTime")); } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org