HBASE-15222 Use less contended classes for metrics Summary: Use less contended things for metrics. For histogram which was the largest culprit we use FastLongHistogram For atomic long where possible we now use counter.
Test Plan: unit tests Reviewers: Subscribers: Differential Revision: https://reviews.facebook.net/D54381 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/630a6582 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/630a6582 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/630a6582 Branch: refs/heads/hbase-12439 Commit: 630a65825ed9a9c00f72bbfcac0588e1ab0cdd72 Parents: 20e14f4 Author: Elliott Clark <ecl...@apache.org> Authored: Thu Feb 18 09:54:05 2016 -0800 Committer: Elliott Clark <ecl...@apache.org> Committed: Wed Feb 24 14:34:05 2016 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/util/FastLongHistogram.java | 162 ++++++++++++++----- .../hbase/util/TestFastLongHistogram.java | 32 ++++ .../apache/hadoop/hbase/metrics/BaseSource.java | 8 - .../apache/hadoop/metrics2/MetricHistogram.java | 3 + hbase-hadoop2-compat/pom.xml | 8 +- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 50 +++--- .../MetricsAssignmentManagerSourceImpl.java | 10 +- .../MetricsMasterFilesystemSourceImpl.java | 14 +- .../hbase/master/MetricsMasterSourceImpl.java | 4 +- .../hbase/master/MetricsSnapshotSourceImpl.java | 8 +- .../balancer/MetricsBalancerSourceImpl.java | 8 +- .../hadoop/hbase/metrics/BaseSourceImpl.java | 17 +- .../MetricsRegionServerSourceImpl.java | 16 +- .../regionserver/MetricsRegionSourceImpl.java | 24 +-- .../regionserver/wal/MetricsWALSourceImpl.java | 10 +- .../MetricsReplicationGlobalSourceSource.java | 36 ++--- .../MetricsReplicationSinkSourceImpl.java | 16 +- .../MetricsReplicationSourceSourceImpl.java | 36 ++--- .../hbase/rest/MetricsRESTSourceImpl.java | 38 ++--- .../thrift/MetricsThriftServerSourceImpl.java | 13 +- .../metrics2/lib/DynamicMetricsRegistry.java | 103 ++---------- .../metrics2/lib/MetricMutableQuantiles.java | 154 ------------------ .../metrics2/lib/MetricsExecutorImpl.java | 2 +- .../hadoop/metrics2/lib/MutableFastCounter.java | 60 +++++++ .../hadoop/metrics2/lib/MutableHistogram.java | 133 +++++---------- .../metrics2/lib/MutableRangeHistogram.java | 75 ++++----- .../metrics2/lib/MutableSizeHistogram.java | 25 ++- .../metrics2/lib/MutableTimeHistogram.java | 23 ++- .../hbase/metrics/TestBaseSourceImpl.java | 5 +- .../tmpl/regionserver/BlockCacheTmpl.jamon | 8 - .../tmpl/regionserver/BlockCacheViewTmpl.jamon | 1 - .../tmpl/regionserver/ServerMetricsTmpl.jamon | 1 - .../hadoop/hbase/io/hfile/AgeSnapshot.java | 38 +++-- .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 31 ++-- .../hadoop/hbase/io/hfile/CacheStats.java | 54 +++---- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 10 +- .../hadoop/hbase/io/hfile/HFileBlock.java | 2 +- .../hadoop/hbase/io/hfile/HFileReaderImpl.java | 4 +- .../hbase/io/hfile/bucket/BucketCacheStats.java | 11 +- .../hbase/regionserver/StoreFileScanner.java | 14 +- 40 files changed, 565 insertions(+), 702 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java index 623cbdb..78b2bf0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.util; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicLongArray; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -31,11 +30,20 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Evolving public class FastLongHistogram { + + /** + * Default number of bins. + */ + public static final int DEFAULT_NBINS = 255; + + public static final double[] DEFAULT_QUANTILES = + new double[]{0.25, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999}; + /** * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. */ private static class Bins { - private final AtomicLongArray counts; + private final Counter[] counts; // inclusive private final long binsMin; // exclusive @@ -43,6 +51,10 @@ public class FastLongHistogram { private final long bins10XMax; private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); private final AtomicLong max = new AtomicLong(0L); + + private final Counter count = new Counter(0); + private final Counter total = new Counter(0); + // set to true when any of data has been inserted to the Bins. It is set after the counts are // updated. private final AtomicBoolean hasData = new AtomicBoolean(false); @@ -50,17 +62,18 @@ public class FastLongHistogram { /** * The constructor for creating a Bins without any prior data. */ - public Bins() { - this.counts = new AtomicLongArray(4); - this.binsMin = 0L; - this.binsMax = Long.MAX_VALUE; - this.bins10XMax = Long.MAX_VALUE; + public Bins(int numBins) { + counts = createCounters(numBins + 3); + this.binsMin = 1L; + + // These two numbers are total guesses + // and should be treated as highly suspect. + this.binsMax = 1000; + this.bins10XMax = binsMax * 10; } - + /** * The constructor for creating a Bins with last Bins. - * @param last the last Bins instance. - * @param quantiles the quantiles for creating the bins of the histogram. */ public Bins(Bins last, int numOfBins, double minQ, double maxQ) { long[] values = last.getQuantiles(new double[] { minQ, maxQ }); @@ -72,28 +85,51 @@ public class FastLongHistogram { this.binsMax = Math.max(binsMax, this.binsMin + numOfBins); this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1); - this.counts = new AtomicLongArray(numOfBins + 3); + this.counts = createCounters(numOfBins + 3); + } + + private Counter[] createCounters(int num) { + Counter[] counters = new Counter[num]; + for (int i = 0; i < num; i++) { + counters[i] = new Counter(); + } + return counters; + } + + private int getIndex(long value) { + if (value < this.binsMin) { + return 0; + } else if (value > this.bins10XMax) { + return this.counts.length - 1; + } else if (value >= this.binsMax) { + return this.counts.length - 2; + } + // compute the position + return 1 + (int) ((value - this.binsMin) * (this.counts.length - 3) / + (this.binsMax - this.binsMin)); + } /** * Adds a value to the histogram. */ public void add(long value, long count) { + if (value < 0) { + // The whole computation is completely thrown off if there are negative numbers + // + // Normally we would throw an IllegalArgumentException however this is the metrics + // system and it should be completely safe at all times. + // So silently throw it away. + return; + } AtomicUtils.updateMin(min, value); AtomicUtils.updateMax(max, value); - if (value < this.binsMin) { - this.counts.addAndGet(0, count); - } else if (value > this.bins10XMax) { - this.counts.addAndGet(this.counts.length() - 1, count); - } else if (value >= this.binsMax) { - this.counts.addAndGet(this.counts.length() - 2, count); - } else { - // compute the position - int pos = - 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin)); - this.counts.addAndGet(pos, count); - } + this.count.add(count); + this.total.add(value * count); + + int pos = getIndex(value); + this.counts[pos].add(count); // hasData needs to be updated as last this.hasData.set(true); @@ -101,7 +137,6 @@ public class FastLongHistogram { /** * Computes the quantiles give the ratios. - * @param smooth set to true to have a prior on the distribution. Used for recreating the bins. */ public long[] getQuantiles(double[] quantiles) { if (!this.hasData.get()) { @@ -112,10 +147,10 @@ public class FastLongHistogram { // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. // This is not synchronized, but since the counter are accumulating, the result is a good // estimation of a snapshot. - long[] counts = new long[this.counts.length()]; + long[] counts = new long[this.counts.length]; long total = 0L; - for (int i = 0; i < this.counts.length(); i++) { - counts[i] = this.counts.get(i); + for (int i = 0; i < this.counts.length; i++) { + counts[i] = this.counts[i].get(); total += counts[i]; } @@ -137,8 +172,8 @@ public class FastLongHistogram { mn = this.binsMax; mx = this.bins10XMax; } else { - mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3); - mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3); + mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length - 3); + mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length - 3); } if (mx < this.min.get()) { @@ -177,12 +212,27 @@ public class FastLongHistogram { return res; } + + + long getNumAtOrBelow(long val) { + final int targetIndex = getIndex(val); + long totalToCurrentIndex = 0; + for (int i = 0; i <= targetIndex; i++) { + totalToCurrentIndex += this.counts[i].get(); + } + return totalToCurrentIndex; + } } // The bins counting values. It is replaced with a new one in calling of reset(). - private volatile Bins bins = new Bins(); - // The quantiles for creating a Bins with last Bins. - private final int numOfBins; + private volatile Bins bins; + + /** + * Constructor. + */ + public FastLongHistogram() { + this(DEFAULT_NBINS); + } /** * Constructor. @@ -190,7 +240,7 @@ public class FastLongHistogram { * results but with lower efficiency, and vice versus. */ public FastLongHistogram(int numOfBins) { - this.numOfBins = numOfBins; + this.bins = new Bins(numOfBins); } /** @@ -202,10 +252,14 @@ public class FastLongHistogram { */ public FastLongHistogram(int numOfBins, long min, long max) { this(numOfBins); - Bins bins = new Bins(); + Bins bins = new Bins(numOfBins); bins.add(min, 1); bins.add(max, 1); - this.bins = new Bins(bins, numOfBins, 0.01, 0.99); + this.bins = new Bins(bins, numOfBins, 0.01, 0.999); + } + + private FastLongHistogram(Bins bins) { + this.bins = bins; } /** @@ -222,12 +276,46 @@ public class FastLongHistogram { return this.bins.getQuantiles(quantiles); } + public long[] getQuantiles() { + return this.bins.getQuantiles(DEFAULT_QUANTILES); + } + + public long getMin() { + return this.bins.min.get(); + } + + public long getMax() { + return this.bins.max.get(); + } + + public long getCount() { + return this.bins.count.get(); + } + + public long getMean() { + Bins bins = this.bins; + long count = bins.count.get(); + long total = bins.total.get(); + if (count == 0) { + return 0; + } + return total / count; + } + + public long getNumAtOrBelow(long value) { + return this.bins.getNumAtOrBelow(value); + } + /** * Resets the histogram for new counting. */ - public void reset() { + public FastLongHistogram reset() { if (this.bins.hasData.get()) { - this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99); + Bins oldBins = this.bins; + this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99); + return new FastLongHistogram(oldBins); } + + return null; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java index f5848f3..d56d143 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java @@ -26,6 +26,8 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertEquals; + /** * Testcases for FastLongHistogram. */ @@ -88,6 +90,36 @@ public class TestFastLongHistogram { } } + + @Test + public void testGetNumAtOrBelow() { + long[] VALUES = { 1, 10, 20, 30, 40, 50 }; + + FastLongHistogram h = new FastLongHistogram(); + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + + h.add(Integer.MAX_VALUE, 1); + + h.reset(); + + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + // Add something way out there to make sure it doesn't throw off the counts. + h.add(Integer.MAX_VALUE, 1); + + assertEquals(100, h.getNumAtOrBelow(1)); + assertEquals(200, h.getNumAtOrBelow(11)); + assertEquals(601, h.getNumAtOrBelow(Long.MAX_VALUE)); + } + + @Test public void testSameValues() { FastLongHistogram hist = new FastLongHistogram(100); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java index 3ab783a..f79aa9f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java @@ -80,14 +80,6 @@ public interface BaseSource { /** - * Add some value to a Quantile (An accurate histogram). - * - * @param name the name of the quantile - * @param value the value to add to the quantile - */ - void updateQuantile(String name, long value); - - /** * Get the metrics context. For hadoop metrics2 system this is usually an all lowercased string. * eg. regionserver, master, thriftserver * http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java index b759efb..9d24a23 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java @@ -30,10 +30,13 @@ public interface MetricHistogram { String MAX_METRIC_NAME = "_max"; String MEAN_METRIC_NAME = "_mean"; String MEDIAN_METRIC_NAME = "_median"; + String TWENTY_FIFTH_PERCENTILE_METRIC_NAME = "_25th_percentile"; String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile"; String NINETIETH_PERCENTILE_METRIC_NAME = "_90th_percentile"; String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile"; + String NINETY_EIGHTH_PERCENTILE_METRIC_NAME = "_98th_percentile"; String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile"; + String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME = "_99.9th_percentile"; /** * Add a single value to a histogram's stream of values. http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index 100a297..56c3e7f 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -162,6 +162,10 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> <artifactId>hbase-hadoop-compat</artifactId> <version>${project.version}</version> <type>test-jar</type> @@ -182,10 +186,6 @@ limitations under the License. <version>${hadoop-two.version}</version> </dependency> <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - </dependency> - <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java index 487f9f5..48f57e9 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java @@ -21,11 +21,11 @@ package org.apache.hadoop.hbase.ipc; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsHBaseServerSourceImpl extends BaseSourceImpl @@ -33,29 +33,29 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl private final MetricsHBaseServerWrapper wrapper; - private final MutableCounterLong authorizationSuccesses; - private final MutableCounterLong authorizationFailures; - private final MutableCounterLong authenticationSuccesses; - private final MutableCounterLong authenticationFailures; - private final MutableCounterLong authenticationFallbacks; - private final MutableCounterLong sentBytes; - private final MutableCounterLong receivedBytes; - - private final MutableCounterLong exceptions; - private final MutableCounterLong exceptionsOOO; - private final MutableCounterLong exceptionsBusy; - private final MutableCounterLong exceptionsUnknown; - private final MutableCounterLong exceptionsSanity; - private final MutableCounterLong exceptionsNSRE; - private final MutableCounterLong exceptionsMoved; - private final MutableCounterLong exceptionsMultiTooLarge; - - - private MutableHistogram queueCallTime; - private MutableHistogram processCallTime; - private MutableHistogram totalCallTime; - private MutableHistogram requestSize; - private MutableHistogram responseSize; + private final MutableFastCounter authorizationSuccesses; + private final MutableFastCounter authorizationFailures; + private final MutableFastCounter authenticationSuccesses; + private final MutableFastCounter authenticationFailures; + private final MutableFastCounter authenticationFallbacks; + private final MutableFastCounter sentBytes; + private final MutableFastCounter receivedBytes; + + private final MutableFastCounter exceptions; + private final MutableFastCounter exceptionsOOO; + private final MutableFastCounter exceptionsBusy; + private final MutableFastCounter exceptionsUnknown; + private final MutableFastCounter exceptionsSanity; + private final MutableFastCounter exceptionsNSRE; + private final MutableFastCounter exceptionsMoved; + private final MutableFastCounter exceptionsMultiTooLarge; + + + private MetricHistogram queueCallTime; + private MetricHistogram processCallTime; + private MetricHistogram totalCallTime; + private MetricHistogram requestSize; + private MetricHistogram responseSize; public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index ccf1c1d..a2192d2 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -20,17 +20,19 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private -public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implements MetricsAssignmentManagerSource { +public class MetricsAssignmentManagerSourceImpl + extends BaseSourceImpl + implements MetricsAssignmentManagerSource { private MutableGaugeLong ritGauge; private MutableGaugeLong ritCountOverThresholdGauge; private MutableGaugeLong ritOldestAgeGauge; - private MutableHistogram assignTimeHisto; - private MutableHistogram bulkAssignTimeHisto; + private MetricHistogram assignTimeHisto; + private MetricHistogram bulkAssignTimeHisto; public MetricsAssignmentManagerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java index 28414ea..7e8542c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java @@ -20,15 +20,17 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; @InterfaceAudience.Private -public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements MetricsMasterFileSystemSource { +public class MetricsMasterFilesystemSourceImpl + extends BaseSourceImpl + implements MetricsMasterFileSystemSource { - private MutableHistogram splitSizeHisto; - private MutableHistogram splitTimeHisto; - private MutableHistogram metaSplitTimeHisto; - private MutableHistogram metaSplitSizeHisto; + private MetricHistogram splitSizeHisto; + private MetricHistogram splitTimeHisto; + private MetricHistogram metaSplitTimeHisto; + private MetricHistogram metaSplitSizeHisto; public MetricsMasterFilesystemSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index b0ba66e..ea21967 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop2 implementation of MetricsMasterSource. @@ -35,7 +35,7 @@ public class MetricsMasterSourceImpl extends BaseSourceImpl implements MetricsMasterSource { private final MetricsMasterWrapper masterWrapper; - private MutableCounterLong clusterRequestsCounter; + private MutableFastCounter clusterRequestsCounter; public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) { this(METRICS_NAME, http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java index c2fc6b9..bcefda0 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java @@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; @InterfaceAudience.Private public class MetricsSnapshotSourceImpl extends BaseSourceImpl implements MetricsSnapshotSource { - private MutableHistogram snapshotTimeHisto; - private MutableHistogram snapshotCloneTimeHisto; - private MutableHistogram snapshotRestoreTimeHisto; + private MetricHistogram snapshotTimeHisto; + private MetricHistogram snapshotCloneTimeHisto; + private MetricHistogram snapshotRestoreTimeHisto; public MetricsSnapshotSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java index da34df2..0a74630 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java @@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master.balancer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsBalancerSourceImpl extends BaseSourceImpl implements MetricsBalancerSource{ - private MutableHistogram blanceClusterHisto; - private MutableCounterLong miscCount; + private MetricHistogram blanceClusterHisto; + private MutableFastCounter miscCount; public MetricsBalancerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java index 6756a21..f843ec2 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java @@ -24,8 +24,7 @@ import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; -import org.apache.hadoop.metrics2.lib.MetricMutableQuantiles; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; import org.apache.hadoop.metrics2.source.JvmMetrics; @@ -88,7 +87,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param value the new value of the gauge. */ public void setGauge(String gaugeName, long value) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, value); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, value); gaugeInt.set(value); } @@ -99,7 +98,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta The amount to increment the gauge by. */ public void incGauge(String gaugeName, long delta) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l); gaugeInt.incr(delta); } @@ -110,7 +109,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta the ammount to subtract from a gauge value. */ public void decGauge(String gaugeName, long delta) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l); gaugeInt.decr(delta); } @@ -121,7 +120,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta the ammount to increment */ public void incCounters(String key, long delta) { - MutableCounterLong counter = metricsRegistry.getLongCounter(key, 0l); + MutableFastCounter counter = metricsRegistry.getCounter(key, 0l); counter.incr(delta); } @@ -132,12 +131,6 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { histo.add(value); } - @Override - public void updateQuantile(String name, long value) { - MetricMutableQuantiles histo = metricsRegistry.getQuantile(name); - histo.add(value); - } - /** * Remove a named gauge. * http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index f669d26..4a2d8cc 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -24,7 +24,7 @@ import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop2 implementation of MetricsRegionServerSource. @@ -45,13 +45,13 @@ public class MetricsRegionServerSourceImpl private final MetricHistogram replayHisto; private final MetricHistogram scanNextHisto; - private final MutableCounterLong slowPut; - private final MutableCounterLong slowDelete; - private final MutableCounterLong slowGet; - private final MutableCounterLong slowIncrement; - private final MutableCounterLong slowAppend; - private final MutableCounterLong splitRequest; - private final MutableCounterLong splitSuccess; + private final MutableFastCounter slowPut; + private final MutableFastCounter slowDelete; + private final MutableFastCounter slowGet; + private final MutableFastCounter slowIncrement; + private final MutableFastCounter slowAppend; + private final MutableFastCounter splitRequest; + private final MutableFastCounter splitSuccess; private final MetricHistogram splitTimeHisto; private final MetricHistogram flushTimeHisto; http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 90c6ce2..c6b7d12 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -23,11 +23,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsRegionSourceImpl implements MetricsRegionSource { @@ -53,12 +53,12 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { private final String regionAppendKey; private final String regionScanNextKey; - private final MutableCounterLong regionPut; - private final MutableCounterLong regionDelete; - private final MutableCounterLong regionIncrement; - private final MutableCounterLong regionAppend; - private final MutableHistogram regionGet; - private final MutableHistogram regionScanNext; + private final MutableFastCounter regionPut; + private final MutableFastCounter regionDelete; + private final MutableFastCounter regionIncrement; + private final MutableFastCounter regionAppend; + private final MetricHistogram regionGet; + private final MetricHistogram regionScanNext; private final int hashCode; public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper, @@ -80,16 +80,16 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { String suffix = "Count"; regionPutKey = regionNamePrefix + MetricsRegionServerSource.MUTATE_KEY + suffix; - regionPut = registry.getLongCounter(regionPutKey, 0L); + regionPut = registry.getCounter(regionPutKey, 0L); regionDeleteKey = regionNamePrefix + MetricsRegionServerSource.DELETE_KEY + suffix; - regionDelete = registry.getLongCounter(regionDeleteKey, 0L); + regionDelete = registry.getCounter(regionDeleteKey, 0L); regionIncrementKey = regionNamePrefix + MetricsRegionServerSource.INCREMENT_KEY + suffix; - regionIncrement = registry.getLongCounter(regionIncrementKey, 0L); + regionIncrement = registry.getCounter(regionIncrementKey, 0L); regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix; - regionAppend = registry.getLongCounter(regionAppendKey, 0L); + regionAppend = registry.getCounter(regionAppendKey, 0L); regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY; regionGet = registry.newTimeHistogram(regionGetKey); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index a149d1b..995e334 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.regionserver.wal; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricHistogram; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** @@ -36,10 +36,10 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo private final MetricHistogram appendSizeHisto; private final MetricHistogram appendTimeHisto; private final MetricHistogram syncTimeHisto; - private final MutableCounterLong appendCount; - private final MutableCounterLong slowAppendCount; - private final MutableCounterLong logRollRequested; - private final MutableCounterLong lowReplicationLogRollRequested; + private final MutableFastCounter appendCount; + private final MutableFastCounter slowAppendCount; + private final MutableFastCounter logRollRequested; + private final MutableFastCounter lowReplicationLogRollRequested; public MetricsWALSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java index 392cd39..93b10b6 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource{ @@ -26,38 +26,38 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS private final MutableGaugeLong ageOfLastShippedOpGauge; private final MutableGaugeLong sizeOfLogQueueGauge; - private final MutableCounterLong logReadInEditsCounter; - private final MutableCounterLong logEditsFilteredCounter; - private final MutableCounterLong shippedBatchesCounter; - private final MutableCounterLong shippedOpsCounter; - private final MutableCounterLong shippedKBsCounter; - private final MutableCounterLong logReadInBytesCounter; - private final MutableCounterLong shippedHFilesCounter; + private final MutableFastCounter logReadInEditsCounter; + private final MutableFastCounter logEditsFilteredCounter; + private final MutableFastCounter shippedBatchesCounter; + private final MutableFastCounter shippedOpsCounter; + private final MutableFastCounter shippedKBsCounter; + private final MutableFastCounter logReadInBytesCounter; + private final MutableFastCounter shippedHFilesCounter; private final MutableGaugeLong sizeOfHFileRefsQueueGauge; public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) { this.rms = rms; - ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); + ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); - sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); + sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); - shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BATCHES, 0L); + shippedBatchesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_BATCHES, 0L); - shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_OPS, 0L); + shippedOpsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_OPS, 0L); - shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L); + shippedKBsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_KBS, 0L); - logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES, 0L); + logReadInBytesCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_BYTES, 0L); - logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS, 0L); + logReadInEditsCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_EDITS, 0L); - logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_EDITS_FILTERED, 0L); + logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_EDITS_FILTERED, 0L); - shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_HFILES, 0L); + shippedHFilesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_HFILES, 0L); sizeOfHFileRefsQueueGauge = - rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L); + rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L); } @Override public void setLastShippedAge(long age) { http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 8f4a337..540212a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -18,21 +18,21 @@ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkSource { private final MutableGaugeLong ageGauge; - private final MutableCounterLong batchesCounter; - private final MutableCounterLong opsCounter; - private final MutableCounterLong hfilesCounter; + private final MutableFastCounter batchesCounter; + private final MutableFastCounter opsCounter; + private final MutableFastCounter hfilesCounter; public MetricsReplicationSinkSourceImpl(MetricsReplicationSourceImpl rms) { - ageGauge = rms.getMetricsRegistry().getLongGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L); - batchesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_BATCHES, 0L); - opsCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_OPS, 0L); - hfilesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_HFILES, 0L); + ageGauge = rms.getMetricsRegistry().getGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L); + batchesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_BATCHES, 0L); + opsCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_OPS, 0L); + hfilesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_HFILES, 0L); } @Override public void setLastAppliedOpAge(long age) { http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java index 217cc3e..9941712 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSourceSource { @@ -37,13 +37,13 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou private final MutableGaugeLong ageOfLastShippedOpGauge; private final MutableGaugeLong sizeOfLogQueueGauge; - private final MutableCounterLong logReadInEditsCounter; - private final MutableCounterLong logEditsFilteredCounter; - private final MutableCounterLong shippedBatchesCounter; - private final MutableCounterLong shippedOpsCounter; - private final MutableCounterLong shippedKBsCounter; - private final MutableCounterLong logReadInBytesCounter; - private final MutableCounterLong shippedHFilesCounter; + private final MutableFastCounter logReadInEditsCounter; + private final MutableFastCounter logEditsFilteredCounter; + private final MutableFastCounter shippedBatchesCounter; + private final MutableFastCounter shippedOpsCounter; + private final MutableFastCounter shippedKBsCounter; + private final MutableFastCounter logReadInBytesCounter; + private final MutableFastCounter shippedHFilesCounter; private final MutableGaugeLong sizeOfHFileRefsQueueGauge; public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id) { @@ -51,34 +51,34 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou this.id = id; ageOfLastShippedOpKey = "source." + id + ".ageOfLastShippedOp"; - ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(ageOfLastShippedOpKey, 0L); + ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(ageOfLastShippedOpKey, 0L); sizeOfLogQueueKey = "source." + id + ".sizeOfLogQueue"; - sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfLogQueueKey, 0L); + sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfLogQueueKey, 0L); shippedBatchesKey = "source." + this.id + ".shippedBatches"; - shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(shippedBatchesKey, 0L); + shippedBatchesCounter = rms.getMetricsRegistry().getCounter(shippedBatchesKey, 0L); shippedOpsKey = "source." + this.id + ".shippedOps"; - shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(shippedOpsKey, 0L); + shippedOpsCounter = rms.getMetricsRegistry().getCounter(shippedOpsKey, 0L); shippedKBsKey = "source." + this.id + ".shippedKBs"; - shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L); + shippedKBsCounter = rms.getMetricsRegistry().getCounter(shippedKBsKey, 0L); logReadInBytesKey = "source." + this.id + ".logReadInBytes"; - logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L); + logReadInBytesCounter = rms.getMetricsRegistry().getCounter(logReadInBytesKey, 0L); logReadInEditsKey = "source." + id + ".logEditsRead"; - logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(logReadInEditsKey, 0L); + logReadInEditsCounter = rms.getMetricsRegistry().getCounter(logReadInEditsKey, 0L); logEditsFilteredKey = "source." + id + ".logEditsFiltered"; - logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(logEditsFilteredKey, 0L); + logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(logEditsFilteredKey, 0L); shippedHFilesKey = "source." + this.id + ".shippedHFiles"; - shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(shippedHFilesKey, 0L); + shippedHFilesCounter = rms.getMetricsRegistry().getCounter(shippedHFilesKey, 0L); sizeOfHFileRefsQueueKey = "source." + id + ".sizeOfHFileRefsQueue"; - sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfHFileRefsQueueKey, 0L); + sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfHFileRefsQueueKey, 0L); } @Override public void setLastShippedAge(long age) { http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java index 9eae18b..71755b7 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.rest; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop Two implementation of a metrics2 source that will export metrics from the Rest server to @@ -31,15 +31,15 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; @InterfaceAudience.Private public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsRESTSource { - private MutableCounterLong request; - private MutableCounterLong sucGet; - private MutableCounterLong sucPut; - private MutableCounterLong sucDel; - private MutableCounterLong sucScan; - private MutableCounterLong fGet; - private MutableCounterLong fPut; - private MutableCounterLong fDel; - private MutableCounterLong fScan; + private MutableFastCounter request; + private MutableFastCounter sucGet; + private MutableFastCounter sucPut; + private MutableFastCounter sucDel; + private MutableFastCounter sucScan; + private MutableFastCounter fGet; + private MutableFastCounter fPut; + private MutableFastCounter fDel; + private MutableFastCounter fScan; public MetricsRESTSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, CONTEXT, JMX_CONTEXT); @@ -55,17 +55,17 @@ public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsREST @Override public void init() { super.init(); - request = getMetricsRegistry().getLongCounter(REQUEST_KEY, 0l); + request = getMetricsRegistry().getCounter(REQUEST_KEY, 0l); - sucGet = getMetricsRegistry().getLongCounter(SUCCESSFUL_GET_KEY, 0l); - sucPut = getMetricsRegistry().getLongCounter(SUCCESSFUL_PUT_KEY, 0l); - sucDel = getMetricsRegistry().getLongCounter(SUCCESSFUL_DELETE_KEY, 0l); - sucScan = getMetricsRegistry().getLongCounter(SUCCESSFUL_SCAN_KEY, 0L); + sucGet = getMetricsRegistry().getCounter(SUCCESSFUL_GET_KEY, 0l); + sucPut = getMetricsRegistry().getCounter(SUCCESSFUL_PUT_KEY, 0l); + sucDel = getMetricsRegistry().getCounter(SUCCESSFUL_DELETE_KEY, 0l); + sucScan = getMetricsRegistry().getCounter(SUCCESSFUL_SCAN_KEY, 0L); - fGet = getMetricsRegistry().getLongCounter(FAILED_GET_KEY, 0l); - fPut = getMetricsRegistry().getLongCounter(FAILED_PUT_KEY, 0l); - fDel = getMetricsRegistry().getLongCounter(FAILED_DELETE_KEY, 0l); - fScan = getMetricsRegistry().getLongCounter(FAILED_SCAN_KEY, 0l); + fGet = getMetricsRegistry().getCounter(FAILED_GET_KEY, 0l); + fPut = getMetricsRegistry().getCounter(FAILED_PUT_KEY, 0l); + fDel = getMetricsRegistry().getCounter(FAILED_DELETE_KEY, 0l); + fScan = getMetricsRegistry().getCounter(FAILED_SCAN_KEY, 0l); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java index f9612e5..f5d83b1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; @@ -32,12 +33,12 @@ import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements MetricsThriftServerSource { - private MutableHistogram batchGetStat; - private MutableHistogram batchMutateStat; - private MutableHistogram queueTimeStat; + private MetricHistogram batchGetStat; + private MetricHistogram batchMutateStat; + private MetricHistogram queueTimeStat; - private MutableHistogram thriftCallStat; - private MutableHistogram thriftSlowCallStat; + private MetricHistogram thriftCallStat; + private MetricHistogram thriftSlowCallStat; private MutableGaugeLong callQueueLenGauge; @@ -56,7 +57,7 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements queueTimeStat = getMetricsRegistry().newTimeHistogram(TIME_IN_QUEUE_KEY); thriftCallStat = getMetricsRegistry().newTimeHistogram(THRIFT_CALL_KEY); thriftSlowCallStat = getMetricsRegistry().newTimeHistogram(SLOW_THRIFT_CALL_KEY); - callQueueLenGauge = getMetricsRegistry().getLongGauge(CALL_QUEUE_LEN_KEY, 0); + callQueueLenGauge = getMetricsRegistry().getGauge(CALL_QUEUE_LEN_KEY, 0); } http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index ee13c76..1f3da12 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -106,35 +106,13 @@ public class DynamicMetricsRegistry { } /** - * Create a mutable integer counter - * @param name of the metric - * @param desc metric description - * @param iVal initial value - * @return a new counter object - */ - public MutableCounterInt newCounter(String name, String desc, int iVal) { - return newCounter(new MetricsInfoImpl(name, desc), iVal); - } - - /** - * Create a mutable integer counter - * @param info metadata of the metric - * @param iVal initial value - * @return a new counter object - */ - public MutableCounterInt newCounter(MetricsInfo info, int iVal) { - MutableCounterInt ret = new MutableCounterInt(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class); - } - - /** * Create a mutable long integer counter * @param name of the metric * @param desc metric description * @param iVal initial value * @return a new counter object */ - public MutableCounterLong newCounter(String name, String desc, long iVal) { + public MutableFastCounter newCounter(String name, String desc, long iVal) { return newCounter(new MetricsInfoImpl(name, desc), iVal); } @@ -144,30 +122,9 @@ public class DynamicMetricsRegistry { * @param iVal initial value * @return a new counter object */ - public MutableCounterLong newCounter(MetricsInfo info, long iVal) { - MutableCounterLong ret = new MutableCounterLong(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class); - } - - /** - * Create a mutable integer gauge - * @param name of the metric - * @param desc metric description - * @param iVal initial value - * @return a new gauge object - */ - public MutableGaugeInt newGauge(String name, String desc, int iVal) { - return newGauge(new MetricsInfoImpl(name, desc), iVal); - } - /** - * Create a mutable integer gauge - * @param info metadata of the metric - * @param iVal initial value - * @return a new gauge object - */ - public MutableGaugeInt newGauge(MetricsInfo info, int iVal) { - MutableGaugeInt ret = new MutableGaugeInt(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class); + public MutableFastCounter newCounter(MetricsInfo info, long iVal) { + MutableFastCounter ret = new MutableFastCounter(info, iVal); + return addNewMetricIfAbsent(info.name(), ret, MutableFastCounter.class); } /** @@ -326,19 +283,6 @@ public class DynamicMetricsRegistry { return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class); } - /** - * Create a new MutableQuantile(A more accurate histogram). - * @param name The name of the histogram - * @return a new MutableQuantile - */ - public MetricMutableQuantiles newQuantile(String name) { - return newQuantile(name, ""); - } - - public MetricMutableQuantiles newQuantile(String name, String desc) { - MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60); - return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class); - } synchronized void add(String name, MutableMetric metric) { addNewMetricIfAbsent(name, metric, MutableMetric.class); @@ -475,7 +419,7 @@ public class DynamicMetricsRegistry { * @param gaugeName name of the gauge to create or get. * @param potentialStartingValue value of the new gauge if we have to create it. */ - public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) { + public MutableGaugeLong getGauge(String gaugeName, long potentialStartingValue) { //Try and get the guage. MutableMetric metric = metricsMap.get(gaugeName); @@ -510,12 +454,12 @@ public class DynamicMetricsRegistry { * @param counterName Name of the counter to get * @param potentialStartingValue starting value if we have to create a new counter */ - public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) { - //See getLongGauge for description on how this works. + public MutableFastCounter getCounter(String counterName, long potentialStartingValue) { + //See getGauge for description on how this works. MutableMetric counter = metricsMap.get(counterName); if (counter == null) { - MutableCounterLong newCounter = - new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue); + MutableFastCounter newCounter = + new MutableFastCounter(new MetricsInfoImpl(counterName, ""), potentialStartingValue); counter = metricsMap.putIfAbsent(counterName, newCounter); if (counter == null) { return newCounter; @@ -523,16 +467,16 @@ public class DynamicMetricsRegistry { } - if (!(counter instanceof MutableCounterLong)) { + if (!(counter instanceof MutableCounter)) { throw new MetricsException("Metric already exists in registry for metric name: " + - counterName + " and not of type MetricMutableCounterLong"); + counterName + " and not of type MutableCounter"); } - return (MutableCounterLong) counter; + return (MutableFastCounter) counter; } public MutableHistogram getHistogram(String histoName) { - //See getLongGauge for description on how this works. + //See getGauge for description on how this works. MutableMetric histo = metricsMap.get(histoName); if (histo == null) { MutableHistogram newCounter = @@ -552,27 +496,6 @@ public class DynamicMetricsRegistry { return (MutableHistogram) histo; } - public MetricMutableQuantiles getQuantile(String histoName) { - //See getLongGauge for description on how this works. - MutableMetric histo = metricsMap.get(histoName); - if (histo == null) { - MetricMutableQuantiles newCounter = - new MetricMutableQuantiles(histoName, "", "Ops", "", 60); - histo = metricsMap.putIfAbsent(histoName, newCounter); - if (histo == null) { - return newCounter; - } - } - - - if (!(histo instanceof MetricMutableQuantiles)) { - throw new MetricsException("Metric already exists in registry for metric name: " + - histoName + " and not of type MutableHistogram"); - } - - return (MetricMutableQuantiles) histo; - } - private<T extends MutableMetric> T addNewMetricIfAbsent(String name, T ret, http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java deleted file mode 100644 index c03654b..0000000 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.lib; - -import static org.apache.hadoop.metrics2.lib.Interns.info; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.metrics2.MetricHistogram; -import org.apache.hadoop.metrics2.MetricsExecutor; -import org.apache.hadoop.metrics2.MetricsInfo; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.util.MetricQuantile; -import org.apache.hadoop.metrics2.util.MetricSampleQuantiles; - -import com.google.common.annotations.VisibleForTesting; - -/** - * Watches a stream of long values, maintaining online estimates of specific quantiles with provably - * low error bounds. This is particularly useful for accurate high-percentile (e.g. 95th, 99th) - * latency metrics. - */ -@InterfaceAudience.Private -public class MetricMutableQuantiles extends MutableMetric implements MetricHistogram { - - static final MetricQuantile[] quantiles = {new MetricQuantile(0.50, 0.050), - new MetricQuantile(0.75, 0.025), new MetricQuantile(0.90, 0.010), - new MetricQuantile(0.95, 0.005), new MetricQuantile(0.99, 0.001)}; - - private final MetricsInfo numInfo; - private final MetricsInfo[] quantileInfos; - private final int interval; - - private MetricSampleQuantiles estimator; - private long previousCount = 0; - private MetricsExecutor executor; - - - @VisibleForTesting - protected Map<MetricQuantile, Long> previousSnapshot = null; - - /** - * Instantiates a new {@link MetricMutableQuantiles} for a metric that rolls itself over on the - * specified time interval. - * - * @param name of the metric - * @param description long-form textual description of the metric - * @param sampleName type of items in the stream (e.g., "Ops") - * @param valueName type of the values - * @param interval rollover interval (in seconds) of the estimator - */ - public MetricMutableQuantiles(String name, String description, String sampleName, - String valueName, int interval) { - String ucName = StringUtils.capitalize(name); - String usName = StringUtils.capitalize(sampleName); - String uvName = StringUtils.capitalize(valueName); - String desc = StringUtils.uncapitalize(description); - String lsName = StringUtils.uncapitalize(sampleName); - String lvName = StringUtils.uncapitalize(valueName); - - numInfo = info(ucName + "Num" + usName, String.format( - "Number of %s for %s with %ds interval", lsName, desc, interval)); - // Construct the MetricsInfos for the quantiles, converting to percentiles - quantileInfos = new MetricsInfo[quantiles.length]; - String nameTemplate = "%s%dthPercentile%dsInterval%s"; - String descTemplate = "%d percentile %s with %d second interval for %s"; - for (int i = 0; i < quantiles.length; i++) { - int percentile = (int) (100 * quantiles[i].quantile); - quantileInfos[i] = info(String.format(nameTemplate, ucName, percentile, interval, uvName), - String.format(descTemplate, percentile, lvName, interval, desc)); - } - - estimator = new MetricSampleQuantiles(quantiles); - executor = new MetricsExecutorImpl(); - this.interval = interval; - executor.getExecutor().scheduleAtFixedRate(new RolloverSample(this), - interval, - interval, - TimeUnit.SECONDS); - } - - @Override - public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) { - if (all || changed()) { - builder.addGauge(numInfo, previousCount); - for (int i = 0; i < quantiles.length; i++) { - long newValue = 0; - // If snapshot is null, we failed to update since the window was empty - if (previousSnapshot != null) { - newValue = previousSnapshot.get(quantiles[i]); - } - builder.addGauge(quantileInfos[i], newValue); - } - if (changed()) { - clearChanged(); - } - } - } - - public synchronized void add(long value) { - estimator.insert(value); - } - - public int getInterval() { - return interval; - } - - /** Runnable used to periodically roll over the internal {@link org.apache.hadoop.metrics2.util.MetricSampleQuantiles} every interval. */ - private static class RolloverSample implements Runnable { - - MetricMutableQuantiles parent; - - public RolloverSample(MetricMutableQuantiles parent) { - this.parent = parent; - } - - @Override - public void run() { - synchronized (parent) { - try { - parent.previousCount = parent.estimator.getCount(); - parent.previousSnapshot = parent.estimator.snapshot(); - } catch (IOException e) { - // Couldn't get a new snapshot because the window was empty - parent.previousCount = 0; - parent.previousSnapshot = null; - } - parent.estimator.clear(); - } - parent.setChanged(); - } - - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java index f70413e..c381609 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.metrics2.MetricsExecutor; /** * Class to handle the ScheduledExecutorService{@link ScheduledExecutorService} used by - * MetricMutableQuantiles{@link MetricMutableQuantiles}, MetricsRegionAggregateSourceImpl, and + * MetricsRegionAggregateSourceImpl, and * JmxCacheBuster */ @InterfaceAudience.Private http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java new file mode 100644 index 0000000..3f6c4a7 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.hbase.util.Counter; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +public class MutableFastCounter extends MutableCounter { + + private final Counter counter; + + protected MutableFastCounter(MetricsInfo info, long iVal) { + super(info); + counter = new Counter(iVal); + } + + @Override + public void incr() { + counter.increment(); + setChanged(); + } + + /** + * Increment the value by a delta + * @param delta of the increment + */ + public void incr(long delta) { + counter.add(delta); + setChanged(); + } + + @Override + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addCounter(info(), value()); + clearChanged(); + } + } + + public long value() { + return counter.get(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index c7ff940..717e0ee 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -18,124 +18,79 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Counter; +import org.apache.hadoop.hbase.util.FastLongHistogram; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import com.codahale.metrics.ExponentiallyDecayingReservoir; -import com.codahale.metrics.Reservoir; -import com.codahale.metrics.Snapshot; - /** * A histogram implementation that runs in constant space, and exports to hadoop2's metrics2 system. */ @InterfaceAudience.Private public class MutableHistogram extends MutableMetric implements MetricHistogram { - - private static final int DEFAULT_SAMPLE_SIZE = 2046; - // the bias towards sampling from more recent data. - // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes - private static final double DEFAULT_ALPHA = 0.015; + // Double buffer the two FastLongHistograms. + // As they are reset they learn how the buckets should be spaced + // So keep two around and use them + protected final FastLongHistogram histogram; protected final String name; protected final String desc; - private final Reservoir reservoir; - private final AtomicLong min; - private final AtomicLong max; - private final AtomicLong sum; - private final AtomicLong count; + protected final Counter counter = new Counter(0); public MutableHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableHistogram(String name, String description) { + this(name, description, Integer.MAX_VALUE << 2); + } + + protected MutableHistogram(String name, String description, long maxExpected) { this.name = StringUtils.capitalize(name); this.desc = StringUtils.uncapitalize(description); - reservoir = new ExponentiallyDecayingReservoir(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA); - count = new AtomicLong(); - min = new AtomicLong(Long.MAX_VALUE); - max = new AtomicLong(Long.MIN_VALUE); - sum = new AtomicLong(); + this.histogram = new FastLongHistogram(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); } public void add(final long val) { - setChanged(); - count.incrementAndGet(); - reservoir.update(val); - setMax(val); - setMin(val); - sum.getAndAdd(val); - } - - private void setMax(final long potentialMax) { - boolean done = false; - while (!done) { - final long currentMax = max.get(); - done = currentMax >= potentialMax - || max.compareAndSet(currentMax, potentialMax); - } - } - - private void setMin(long potentialMin) { - boolean done = false; - while (!done) { - final long currentMin = min.get(); - done = currentMin <= potentialMin - || min.compareAndSet(currentMin, potentialMin); - } - } - - public long getMax() { - if (count.get() > 0) { - return max.get(); - } - return 0L; - } - - public long getMin() { - if (count.get() > 0) { - return min.get(); - } - return 0L; - } - - public double getMean() { - long cCount = count.get(); - if (cCount > 0) { - return sum.get() / (double) cCount; - } - return 0.0; + counter.increment(); + histogram.add(val, 1); } @Override - public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - if (all || changed()) { - clearChanged(); - updateSnapshotMetrics(metricsRecordBuilder); - } + public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + FastLongHistogram histo = histogram.reset(); + updateSnapshotMetrics(metricsRecordBuilder, histo); } - public void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder) { - final Snapshot s = reservoir.getSnapshot(); - metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), count.get()); - - metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), getMin()); - metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), getMax()); - metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), getMean()); - - metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian()); - metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), - s.get75thPercentile()); - metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), - s.getValue(0.90)); - metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), - s.get95thPercentile()); - metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), - s.get99thPercentile()); + protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder, + FastLongHistogram histo) { + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), counter.get()); + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), histo.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), histo.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), histo.getMean()); + + long[] percentiles = histo.getQuantiles(); + + metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[0]); + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), + percentiles[1]); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[2]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), + percentiles[3]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[4]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc), + percentiles[5]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + percentiles[6]); + metricsRecordBuilder.addGauge( + Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc), + percentiles[7]); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java index ac1f497..ac8aee0 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java @@ -18,26 +18,29 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLongArray; - import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; - /** * Extended histogram implementation with metric range counters. */ @InterfaceAudience.Private -public abstract class MutableRangeHistogram extends MutableHistogram { +public abstract class MutableRangeHistogram extends MutableHistogram implements MetricHistogram { public MutableRangeHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableRangeHistogram(String name, String description) { - super(name, description); + this(name, description, Integer.MAX_VALUE << 2); } - + + public MutableRangeHistogram(String name, String description, long expectedMax) { + super(name, description, expectedMax); + } + /** * Returns the type of range histogram size or time */ @@ -46,49 +49,39 @@ public abstract class MutableRangeHistogram extends MutableHistogram { /** * Returns the ranges to be counted */ - public abstract long[] getRange(); - - /** - * Returns the range counts - */ - public abstract AtomicLongArray getRangeVals(); + public abstract long[] getRanges(); - @Override - public void add(final long val) { - super.add(val); - updateBand(val); - } - - private void updateBand(final long val) { - int i; - for (i=0; i<getRange().length && val > getRange()[i]; i++); - getRangeVals().incrementAndGet(i); - } @Override - public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - if (all || changed()) { - clearChanged(); - updateSnapshotMetrics(metricsRecordBuilder); - updateSnapshotRangeMetrics(metricsRecordBuilder); - } + public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + FastLongHistogram histo = histogram.reset(); + updateSnapshotMetrics(metricsRecordBuilder, histo); + updateSnapshotRangeMetrics(metricsRecordBuilder, histo); } - - public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder) { - long prior = 0; - for (int i = 0; i < getRange().length; i++) { - long val = getRangeVals().get(i); - if (val > 0) { + + public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder, + FastLongHistogram histogram) { + long priorRange = 0; + long cumNum = 0; + + final long[] ranges = getRanges(); + final String rangeType = getRangeType(); + for (int i = 0; i < ranges.length - 1; i++) { + long val = histogram.getNumAtOrBelow(ranges[i]); + if (val - cumNum > 0) { metricsRecordBuilder.addCounter( - Interns.info(name + "_" + getRangeType() + "_" + prior + "-" + getRange()[i], desc), val); + Interns.info(name + "_" + rangeType + "_" + priorRange + "-" + ranges[i], desc), + val - cumNum); } - prior = getRange()[i]; + priorRange = ranges[i]; + cumNum = val; } - long val = getRangeVals().get(getRange().length); - if (val > 0) { + long val = histogram.getCount(); + if (val - cumNum > 0) { metricsRecordBuilder.addCounter( - Interns.info(name + "_" + getRangeType() + "_" + getRange()[getRange().length - 1] + "-inf", desc), - getRangeVals().get(getRange().length)); + Interns.info(name + "_" + rangeType + "_" + ranges[ranges.length - 1] + "-inf", desc), + val - cumNum); } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/630a6582/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java index 2f1d57a..38e78a2 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java @@ -18,8 +18,6 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLongArray; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; @@ -28,30 +26,29 @@ import org.apache.hadoop.metrics2.MetricsInfo; */ @InterfaceAudience.Private public class MutableSizeHistogram extends MutableRangeHistogram { - private final String rangeType = "SizeRangeCount"; - private final long[] ranges = {10,100,1000,10000,100000,1000000,10000000,100000000}; - private final AtomicLongArray rangeVals = new AtomicLongArray(getRange().length+1); + private final static String RANGE_TYPE = "SizeRangeCount"; + private final static long[] RANGES = {10,100,1000,10000,100000,1000000,10000000,100000000}; public MutableSizeHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableSizeHistogram(String name, String description) { - super(name, description); + this(name, description, RANGES[RANGES.length-2]); + } + + public MutableSizeHistogram(String name, String description, long expectedMax) { + super(name, description, expectedMax); } @Override public String getRangeType() { - return rangeType; + return RANGE_TYPE; } @Override - public long[] getRange() { - return ranges; + public long[] getRanges() { + return RANGES; } - - @Override - public AtomicLongArray getRangeVals() { - return rangeVals; - } + }