Repository: hbase Updated Branches: refs/heads/branch-1.4 decf9d623 -> 52970c05d
HBASE-18652 Expose individual cache stats in a CombinedCache through JMX (Biju Nair) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52970c05 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52970c05 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52970c05 Branch: refs/heads/branch-1.4 Commit: 52970c05da1f63be77ed09b614a06c2fd1aed0ea Parents: decf9d6 Author: Michael Stack <st...@apache.org> Authored: Mon Sep 25 20:51:11 2017 -0700 Committer: Michael Stack <st...@apache.org> Committed: Mon Sep 25 20:56:45 2017 -0700 ---------------------------------------------------------------------- .../regionserver/MetricsRegionServerSource.java | 17 +++++ .../MetricsRegionServerWrapper.java | 40 +++++++++++ .../MetricsRegionServerSourceImpl.java | 16 +++++ .../hadoop/hbase/io/hfile/CacheConfig.java | 37 ++++++++-- .../MetricsRegionServerWrapperImpl.java | 71 +++++++++++++++++++- .../hbase/io/hfile/TestBlockCacheReporting.java | 2 +- .../hadoop/hbase/io/hfile/TestCacheConfig.java | 4 +- .../io/hfile/TestForceCacheImportantBlocks.java | 2 +- .../MetricsRegionServerWrapperStub.java | 40 +++++++++++ .../regionserver/TestMetricsRegionServer.java | 8 +++ 10 files changed, 226 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 3ac678e..b72deb8 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -345,6 +345,23 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount"; String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount"; + String L1_CACHE_HIT_COUNT = "l1CacheHitCount"; + String L1_CACHE_HIT_COUNT_DESC = "L1 cache hit count."; + String L1_CACHE_MISS_COUNT = "l1CacheMissCount"; + String L1_CACHE_MISS_COUNT_DESC = "L1 cache miss count."; + String L1_CACHE_HIT_RATIO = "l1CacheHitRatio"; + String L1_CACHE_HIT_RATIO_DESC = "L1 cache hit ratio."; + String L1_CACHE_MISS_RATIO = "l1CacheMissRatio"; + String L1_CACHE_MISS_RATIO_DESC = "L1 cache miss ratio."; + String L2_CACHE_HIT_COUNT = "l2CacheHitCount"; + String L2_CACHE_HIT_COUNT_DESC = "L2 cache hit count."; + String L2_CACHE_MISS_COUNT = "l2CacheMissCount"; + String L2_CACHE_MISS_COUNT_DESC = "L2 cache miss count."; + String L2_CACHE_HIT_RATIO = "l2CacheHitRatio"; + String L2_CACHE_HIT_RATIO_DESC = "L2 cache hit ratio."; + String L2_CACHE_MISS_RATIO = "l2CacheMissRatio"; + String L2_CACHE_MISS_RATIO_DESC = "L2 cache miss ratio."; + String RS_START_TIME_NAME = "regionServerStartTime"; String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum"; String SERVER_NAME_NAME = "serverName"; http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 7d7f66d..3344dce 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -259,6 +259,46 @@ public interface MetricsRegionServerWrapper { long getBlockCacheFailedInsertions(); /** + * Hit count of L1 cache. + */ + public long getL1CacheHitCount(); + + /** + * Miss count of L1 cache. + */ + public long getL1CacheMissCount(); + + /** + * Hit ratio of L1 cache. + */ + public double getL1CacheHitRatio(); + + /** + * Miss ratio of L1 cache. + */ + public double getL1CacheMissRatio(); + + /** + * Hit count of L2 cache. + */ + public long getL2CacheHitCount(); + + /** + * Miss count of L2 cache. + */ + public long getL2CacheMissCount(); + + /** + * Hit ratio of L2 cache. + */ + public double getL2CacheHitRatio(); + + /** + * Miss ratio of L2 cache. + */ + public double getL2CacheMissRatio(); + + /** * Force a re-computation of the metrics. */ void forceRecompute(); http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 628a564..208188e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -458,6 +458,22 @@ public class MetricsRegionServerSourceImpl .addCounter(Interns.info(BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT, ""), rsWrap.getDeleteFamilyBloomHitCount()) .addCounter(Interns.info(BLOCK_CACHE_TRAILER_HIT_COUNT, ""), rsWrap.getTrailerHitCount()) + .addGauge(Interns.info(L1_CACHE_HIT_COUNT, L1_CACHE_HIT_COUNT_DESC), + rsWrap.getL1CacheHitCount()) + .addGauge(Interns.info(L1_CACHE_MISS_COUNT, L1_CACHE_MISS_COUNT_DESC), + rsWrap.getL1CacheMissCount()) + .addGauge(Interns.info(L1_CACHE_HIT_RATIO, L1_CACHE_HIT_RATIO_DESC), + rsWrap.getL1CacheHitRatio()) + .addGauge(Interns.info(L1_CACHE_MISS_RATIO, L1_CACHE_MISS_RATIO_DESC), + rsWrap.getL1CacheMissRatio()) + .addGauge(Interns.info(L2_CACHE_HIT_COUNT, L2_CACHE_HIT_COUNT_DESC), + rsWrap.getL2CacheHitCount()) + .addGauge(Interns.info(L2_CACHE_MISS_COUNT, L2_CACHE_MISS_COUNT_DESC), + rsWrap.getL2CacheMissCount()) + .addGauge(Interns.info(L2_CACHE_HIT_RATIO, L2_CACHE_HIT_RATIO_DESC), + rsWrap.getL2CacheHitRatio()) + .addGauge(Interns.info(L2_CACHE_MISS_RATIO, L2_CACHE_MISS_RATIO_DESC), + rsWrap.getL2CacheMissRatio()) .addCounter(Interns.info(UPDATES_BLOCKED_TIME, UPDATES_BLOCKED_DESC), rsWrap.getUpdatesBlockedTime()) .addCounter(Interns.info(FLUSHED_CELLS, FLUSHED_CELLS_DESC), http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 1d68b99..d020eb0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -525,6 +525,8 @@ public class CacheConfig { // Clear this if in tests you'd make more than one block cache instance. @VisibleForTesting static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE; + private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE = null; + private static BlockCache GLOBAL_L2_CACHE_INSTANCE = null; /** Boolean whether we have disabled the block cache entirely. */ @VisibleForTesting @@ -534,7 +536,8 @@ public class CacheConfig { * @param c Configuration to use. * @return An L1 instance. Currently an instance of LruBlockCache. */ - private static LruBlockCache getL1(final Configuration c) { + private static synchronized LruBlockCache getL1(final Configuration c) { + if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE; final long lruCacheSize = HeapMemorySizeUtil.getLruCacheSize(c); if (lruCacheSize < 0) { blockCacheDisabled = true; @@ -543,7 +546,8 @@ public class CacheConfig { int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE); LOG.info("Allocating LruBlockCache size=" + StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize)); - return new LruBlockCache(lruCacheSize, blockSize, true, c); + GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c); + return GLOBAL_L1_CACHE_INSTANCE; } /** @@ -560,10 +564,26 @@ public class CacheConfig { // If we want to use an external block cache then create that. if (useExternal) { - return getExternalBlockcache(c); + GLOBAL_L2_CACHE_INSTANCE = getExternalBlockcache(c); + } else { + // otherwise use the bucket cache. + GLOBAL_L2_CACHE_INSTANCE = getBucketCache(c); + } + return GLOBAL_L2_CACHE_INSTANCE; + } + + public CacheStats getL1Stats() { + if (GLOBAL_L1_CACHE_INSTANCE != null) { + return GLOBAL_L1_CACHE_INSTANCE.getStats(); } - // otherwise use the bucket cache. - return getBucketCache(c); + return null; + } + + public CacheStats getL2Stats() { + if (GLOBAL_L2_CACHE_INSTANCE != null) { + return GLOBAL_L2_CACHE_INSTANCE.getStats(); + } + return null; } private static BlockCache getExternalBlockcache(Configuration c) { @@ -682,4 +702,11 @@ public class CacheConfig { } return GLOBAL_BLOCK_CACHE_INSTANCE; } + + @VisibleForTesting + static synchronized void clearGlobalInstances() { + GLOBAL_L1_CACHE_INSTANCE = null; + GLOBAL_L2_CACHE_INSTANCE = null; + GLOBAL_BLOCK_CACHE_INSTANCE = null; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index e805e75..31cfbd1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -84,6 +84,8 @@ class MetricsRegionServerWrapperImpl private volatile long averageRegionSize = 0L; private CacheStats cacheStats; + private CacheStats l1Stats = null; + private CacheStats l2Stats = null; private ScheduledExecutorService executor; private Runnable runnable; private long period; @@ -113,8 +115,12 @@ class MetricsRegionServerWrapperImpl */ private synchronized void initBlockCache() { CacheConfig cacheConfig = this.regionServer.cacheConfig; - if (cacheConfig != null && this.blockCache == null) { - this.blockCache = cacheConfig.getBlockCache(); + if (cacheConfig != null) { + l1Stats = cacheConfig.getL1Stats(); + l2Stats = cacheConfig.getL2Stats(); + if (this.blockCache == null) { + this.blockCache = cacheConfig.getBlockCache(); + } } if (this.blockCache != null && this.cacheStats == null) { @@ -328,6 +334,67 @@ class MetricsRegionServerWrapperImpl return this.cacheStats.getFailedInserts(); } + @Override + public long getL1CacheHitCount() { + return 200; + } + + @Override + public long getL1CacheMissCount() { + if (this.l1Stats == null) { + return 0; + } + return this.l1Stats.getMissCount(); + } + + @Override + public double getL1CacheHitRatio() { + if (this.l1Stats == null) { + return 0; + } + return this.l1Stats.getHitRatio(); + } + + @Override + public double getL1CacheMissRatio() { + if (this.l1Stats == null) { + return 0; + } + return this.l1Stats.getMissRatio(); + } + + @Override + public long getL2CacheHitCount() { + if (this.l2Stats == null) { + return 0; + } + return this.l2Stats.getHitCount(); + } + + @Override + public long getL2CacheMissCount() { + if (this.l2Stats == null) { + return 0; + } + return this.l2Stats.getMissCount(); + } + + @Override + public double getL2CacheHitRatio() { + if (this.l2Stats == null) { + return 0; + } + return this.l2Stats.getHitRatio(); + } + + @Override + public double getL2CacheMissRatio() { + if (this.l2Stats == null) { + return 0; + } + return this.l2Stats.getMissRatio(); + } + @Override public void forceRecompute() { this.runnable.run(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index 7ce9679..b8fe745 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -45,7 +45,7 @@ public class TestBlockCacheReporting { @Before public void setUp() throws Exception { - CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null; + CacheConfig.clearGlobalInstances(); this.conf = HBaseConfiguration.create(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index 3f60cd7..eea6558 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -150,14 +150,14 @@ public class TestCacheConfig { @Before public void setUp() throws Exception { - CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null; + CacheConfig.clearGlobalInstances(); this.conf = HBaseConfiguration.create(); } @After public void tearDown() throws Exception { // Let go of current block cache. - CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null; + CacheConfig.clearGlobalInstances(); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java index 16583d9..6c4585a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java @@ -97,7 +97,7 @@ public class TestForceCacheImportantBlocks { @Before public void setup() { // Make sure we make a new one each time. - CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null; + CacheConfig.clearGlobalInstances(); HFile.DATABLOCK_READ_COUNT.set(0); } http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index b573323..21d06cf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -261,6 +261,46 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe } @Override + public long getL1CacheHitCount() { + return 200; + } + + @Override + public long getL1CacheMissCount() { + return 100; + } + + @Override + public double getL1CacheHitRatio() { + return 80; + } + + @Override + public double getL1CacheMissRatio() { + return 20; + } + + @Override + public long getL2CacheHitCount() { + return 800; + } + + @Override + public long getL2CacheMissCount() { + return 200; + } + + @Override + public double getL2CacheHitRatio() { + return 90; + } + + @Override + public double getL2CacheMissRatio() { + return 10; + } + + @Override public long getUpdatesBlockedTime() { return 419; } http://git-wip-us.apache.org/repos/asf/hbase/blob/52970c05/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index e1ef971..92acad4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -97,6 +97,14 @@ public class TestMetricsRegionServer { HELPER.assertGauge("blockCacheCountHitPercent", 98, serverSource); HELPER.assertGauge("blockCacheExpressHitPercent", 97, serverSource); HELPER.assertCounter("blockCacheFailedInsertionCount", 36, serverSource); + HELPER.assertGauge("l1CacheHitCount", 200, serverSource); + HELPER.assertGauge("l1CacheMissCount", 100, serverSource); + HELPER.assertGauge("l1CacheHitRatio", 80, serverSource); + HELPER.assertGauge("l1CacheMissRatio", 20, serverSource); + HELPER.assertGauge("l2CacheHitCount", 800, serverSource); + HELPER.assertGauge("l2CacheMissCount", 200, serverSource); + HELPER.assertGauge("l2CacheHitRatio", 90, serverSource); + HELPER.assertGauge("l2CacheMissRatio", 10, serverSource); HELPER.assertCounter("updatesBlockedTime", 419, serverSource); }