Repository: hbase Updated Branches: refs/heads/branch-1.3 ca11c5870 -> 323768148
HBASE-17017 Remove the current per-region latency histogram metrics Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/32376814 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/32376814 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/32376814 Branch: refs/heads/branch-1.3 Commit: 323768148ab062ae50ae22f4a60fe2c58e05ec2d Parents: ca11c58 Author: Enis Soztutar <e...@apache.org> Authored: Tue Nov 8 15:43:41 2016 -0800 Committer: Enis Soztutar <e...@apache.org> Committed: Mon Nov 14 17:27:21 2016 -0800 ---------------------------------------------------------------------- .../regionserver/MetricsRegionServerSource.java | 5 +- .../hbase/regionserver/MetricsRegionSource.java | 12 ----- .../regionserver/MetricsRegionSourceImpl.java | 52 ++++++-------------- .../hadoop/hbase/regionserver/HRegion.java | 7 +-- .../hbase/regionserver/MetricsRegion.java | 8 --- .../hbase/regionserver/RSRpcServices.java | 1 - .../regionserver/TestRegionServerMetrics.java | 12 ++--- 7 files changed, 25 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 182122a..1991105 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -316,7 +316,7 @@ public interface MetricsRegionServerSource extends BaseSource { String BLOCK_CACHE_GENERAL_BLOOM_META_HIT_COUNT = "blockCacheGeneralBloomMetaHitCount"; String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount"; String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount"; - + String RS_START_TIME_NAME = "regionServerStartTime"; String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum"; String SERVER_NAME_NAME = "serverName"; @@ -335,6 +335,7 @@ public interface MetricsRegionServerSource extends BaseSource { String MUTATE_KEY = "mutate"; String APPEND_KEY = "append"; String REPLAY_KEY = "replay"; + String SCAN_KEY = "scan"; String SCAN_SIZE_KEY = "scanSize"; String SCAN_TIME_KEY = "scanTime"; @@ -446,6 +447,6 @@ public interface MetricsRegionServerSource extends BaseSource { String RPC_MUTATE_REQUEST_COUNT_DESC = "Number of rpc mutation requests this region server has answered."; String AVERAGE_REGION_SIZE = "averageRegionSize"; - String AVERAGE_REGION_SIZE_DESC = + String AVERAGE_REGION_SIZE_DESC = "Average region size over the region server including memstore and storefile sizes."; } http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java index 12ef07c..decf841 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java @@ -60,24 +60,12 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> { void updateDelete(); /** - * Update count and sizes of gets. - * @param getSize size in bytes of the resulting key values for a get - */ - void updateGetSize(long getSize); - - /** * Update time of gets * @param mills time for this get operation. */ void updateGet(long mills); /** - * Update the count and sizes of resultScanner.next() - * @param scanSize Size in bytes of the resulting key values for a next() - */ - void updateScanSize(long scanSize); - - /** * Update time used of resultScanner.next(). * */ void updateScanTime(long mills); http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 16f36bb..8f17e93 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.Interns; -import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableFastCounter; @@ -48,21 +47,22 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { private final String regionNamePrefix; private final String regionPutKey; private final String regionDeleteKey; - private final String regionGetSizeKey; private final String regionGetKey; private final String regionIncrementKey; private final String regionAppendKey; - private final String regionScanSizeKey; - private final String regionScanTimeKey; + private final String regionScanKey; + /* + * Implementation note: Do not put histograms per region. With hundreds of regions in a server + * histograms allocate too many counters. See HBASE-17016. + */ private final MutableFastCounter regionPut; private final MutableFastCounter regionDelete; private final MutableFastCounter regionIncrement; private final MutableFastCounter regionAppend; - private final MetricHistogram regionGetSize; - private final MetricHistogram regionGet; - private final MetricHistogram regionScanSize; - private final MetricHistogram regionScanTime; + private final MutableFastCounter regionGet; + private final MutableFastCounter regionScan; + private final int hashCode; public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper, @@ -95,17 +95,11 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix; regionAppend = registry.getCounter(regionAppendKey, 0L); - regionGetSizeKey = regionNamePrefix + MetricsRegionServerSource.GET_SIZE_KEY; - regionGetSize = registry.newSizeHistogram(regionGetSizeKey); - - regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY; - regionGet = registry.newTimeHistogram(regionGetKey); - - regionScanSizeKey = regionNamePrefix + MetricsRegionServerSource.SCAN_SIZE_KEY; - regionScanSize = registry.newSizeHistogram(regionScanSizeKey); + regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY + suffix; + regionGet = registry.getCounter(regionGetKey, 0L); - regionScanTimeKey = regionNamePrefix + MetricsRegionServerSource.SCAN_TIME_KEY; - regionScanTime = registry.newTimeHistogram(regionScanTimeKey); + regionScanKey = regionNamePrefix + MetricsRegionServerSource.SCAN_KEY + suffix; + regionScan = registry.getCounter(regionScanKey, 0L); hashCode = regionWrapper.getRegionHashCode(); } @@ -134,14 +128,8 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { registry.removeMetric(regionDeleteKey); registry.removeMetric(regionIncrementKey); registry.removeMetric(regionAppendKey); - registry.removeMetric(regionGetSizeKey); registry.removeMetric(regionGetKey); - registry.removeMetric(regionScanSizeKey); - registry.removeMetric(regionScanTimeKey); - registry.removeHistogramMetrics(regionGetSizeKey); - registry.removeHistogramMetrics(regionGetKey); - registry.removeHistogramMetrics(regionScanSizeKey); - registry.removeHistogramMetrics(regionScanTimeKey); + registry.removeMetric(regionScanKey); regionWrapper = null; } @@ -158,23 +146,13 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { } @Override - public void updateGetSize(long getSize) { - regionGetSize.add(getSize); - } - - @Override public void updateGet(long mills) { - regionGet.add(mills); - } - - @Override - public void updateScanSize(long scanSize) { - regionScanSize.add(scanSize); + regionGet.incr(); } @Override public void updateScanTime(long mills) { - regionScanTime.add(mills); + regionScan.incr(); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 89aed2d..2b5e9a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -252,7 +252,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi */ protected volatile long lastReplayedOpenRegionSeqId = -1L; protected volatile long lastReplayedCompactionSeqId = -1L; - + // collects Map(s) of Store to sequence Id when handleFileNotFound() is involved protected List<Map> storeSeqIds = new ArrayList<>(); @@ -7044,11 +7044,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi void metricsUpdateForGet(List<Cell> results, long before) { if (this.metricsRegion != null) { - long totalSize = 0L; - for (Cell cell : results) { - totalSize += CellUtil.estimatedSerializedSizeOf(cell); - } - this.metricsRegion.updateGetSize(totalSize); this.metricsRegion.updateGet(EnvironmentEdgeManager.currentTime() - before); } http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java index 94be034..0364e91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegion.java @@ -49,18 +49,10 @@ public class MetricsRegion { source.updateDelete(); } - public void updateGetSize(final long getSize) { - source.updateGetSize(getSize); - } - public void updateGet(final long t) { source.updateGet(t); } - public void updateScanSize(final long scanSize) { - source.updateScanSize(scanSize); - } - public void updateScanTime(final long t) { source.updateScanTime(t); } http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 4a9e615..335eab2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -2718,7 +2718,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler, region.updateReadRequestsCount(i); long end = EnvironmentEdgeManager.currentTime(); long responseCellSize = context != null ? context.getResponseCellSize() : 0; - region.getMetrics().updateScanSize(responseCellSize); region.getMetrics().updateScanTime(end - before); if (regionServer.metricsRegionServer != null) { regionServer.metricsRegionServer.updateScanSize(responseCellSize); http://git-wip-us.apache.org/repos/asf/hbase/blob/32376814/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index d068217..e2ca677 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -157,7 +157,7 @@ public class TestRegionServerMetrics { "_table_"+tableNameString + "_region_" + i.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg); + metricsHelper.assertCounter(prefix + "_getCount", 10, agg); metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg); } } @@ -229,8 +229,8 @@ public class TestRegionServerMetrics { "_table_"+tableNameString + "_region_" + i.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_getSizeNumOps", 10, agg); - metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg); + metricsHelper.assertCounter(prefix + "_getCount", 10, agg); + metricsHelper.assertCounter(prefix + "_getCount", 10, agg); } metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource); } @@ -446,7 +446,7 @@ public class TestRegionServerMetrics { "_table_"+tableNameString + "_region_" + i.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg); + metricsHelper.assertCounter(prefix + "_scanCount", NUM_SCAN_NEXT, agg); } metricsHelper.assertCounterGt("ScanSize_num_ops", numScanNext, serverSource); } @@ -496,7 +496,7 @@ public class TestRegionServerMetrics { "_table_"+tableNameString + "_region_" + i.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_scanTimeNumOps", NUM_SCAN_NEXT, agg); + metricsHelper.assertCounter(prefix + "_scanCount", NUM_SCAN_NEXT, agg); } metricsHelper.assertCounterGt("ScanTime_num_ops", numScanNext, serverSource); } @@ -548,7 +548,7 @@ public class TestRegionServerMetrics { "_table_"+tableNameString + "_region_" + i.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg); + metricsHelper.assertCounter(prefix + "_scanCount", NUM_SCAN_NEXT, agg); } metricsHelper.assertCounterGt("ScanSize_num_ops", numScanNext, serverSource); }
