Repository: hbase Updated Branches: refs/heads/0.98 34a7ebe01 -> 54e397049
HBASE-15412 Add average region size metric (Alicia Ying Shu) Conflicts: hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java Amending-Author: Andrew Purtell <apurt...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/54e39704 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/54e39704 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/54e39704 Branch: refs/heads/0.98 Commit: 54e39704978ec58195333ced32a64708f4c97608 Parents: 34a7ebe Author: Enis Soztutar <e...@apache.org> Authored: Tue Mar 22 14:46:27 2016 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Tue Mar 29 18:38:37 2016 -0700 ---------------------------------------------------------------------- .../regionserver/MetricsRegionServerSource.java | 4 ++++ .../MetricsRegionServerWrapper.java | 7 ++++++- .../MetricsRegionServerSourceImpl.java | 1 + .../MetricsRegionServerSourceImpl.java | 2 +- .../MetricsRegionServerWrapperImpl.java | 14 ++++++++++++- .../MetricsRegionServerWrapperStub.java | 5 +++++ .../regionserver/TestRegionServerMetrics.java | 21 ++++++++++++++++++++ 7 files changed, 51 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 7233e0e..33728fd 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -286,4 +286,8 @@ public interface MetricsRegionServerSource extends BaseSource { String SPLIT_SUCCESS_KEY = "splitSuccessCount"; String SPLIT_SUCCESS_DESC = "Number of successfully executed splits"; String FLUSH_KEY = "flushTime"; + + String AVERAGE_REGION_SIZE = "averageRegionSize"; + String AVERAGE_REGION_SIZE_DESC = + "Average region size over the region server including memstore and storefile sizes."; } http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 598b043..e0fc7bb 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -280,5 +280,10 @@ public interface MetricsRegionServerWrapper { /** * @return Count of requests blocked because the memstore size is larger than blockingMemStoreSize */ - public long getBlockedRequestsCount(); + long getBlockedRequestsCount(); + + /** + * Get the average region size to this region server. + */ + long getAverageRegionSize(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 0c48976..4f9dd65 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -196,6 +196,7 @@ public class MetricsRegionServerSourceImpl .addGauge(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC, rsWrap.getAvgStoreFileAge()) .addGauge(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC, rsWrap.getNumReferenceFiles()) .addGauge(RS_START_TIME_NAME, RS_START_TIME_DESC, rsWrap.getStartCode()) + .addGauge(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC, rsWrap.getAverageRegionSize()) .addCounter(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC, rsWrap.getTotalRequestCount()) .addCounter(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC, rsWrap.getReadRequestsCount()) .addCounter(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC, rsWrap.getWriteRequestsCount()) http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 5c8189b..aaff3d2 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -205,6 +205,7 @@ public class MetricsRegionServerSourceImpl rsWrap.getNumReferenceFiles()) .addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC), rsWrap.getStartCode()) + .addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize()) .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), rsWrap.getTotalRequestCount()) .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC), @@ -266,7 +267,6 @@ public class MetricsRegionServerSourceImpl rsWrap.getCompactedCellsSize()) .addCounter(Interns.info(MAJOR_COMPACTED_CELLS_SIZE, MAJOR_COMPACTED_CELLS_SIZE_DESC), rsWrap.getMajorCompactedCellsSize()) - .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), rsWrap.getBlockedRequestsCount()) http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 98d80fc..cdc032f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -77,6 +77,7 @@ class MetricsRegionServerWrapperImpl private volatile long compactedCellsSize = 0; private volatile long majorCompactedCellsSize = 0; private volatile long blockedRequestsCount = 0L; + private volatile long averageRegionSize = 0L; private CacheStats cacheStats; private ScheduledExecutorService executor; @@ -472,6 +473,7 @@ class MetricsRegionServerWrapperImpl long tempMajorCompactedCellsSize = 0; long tempBlockedRequestsCount = 0L; + int regionCount = 0; for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get(); tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get(); @@ -511,6 +513,7 @@ class MetricsRegionServerWrapperImpl } hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution()); + regionCount++; } float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex( @@ -527,7 +530,6 @@ class MetricsRegionServerWrapperImpl lastRan = currentTime - period; } - //If we've time traveled keep the last requests per second. if ((currentTime - lastRan) > 0) { long currentRequestCount = getTotalRequestCount(); @@ -555,6 +557,10 @@ class MetricsRegionServerWrapperImpl memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; maxStoreFileAge = tempMaxStoreFileAge; + if (regionCount > 0) { + averageRegionSize = (memstoreSize + storeFileSize) / regionCount; + } + if (tempMinStoreFileAge != Long.MAX_VALUE) { minStoreFileAge = tempMinStoreFileAge; } @@ -588,4 +594,10 @@ class MetricsRegionServerWrapperImpl public long getBlockedRequestsCount() { return blockedRequestsCount; } + + @Override + public long getAverageRegionSize() { + return averageRegionSize; + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 0b4bbb1..cd20532 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -274,4 +274,9 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe public int getSplitQueueSize() { return 0; } + + @Override + public long getAverageRegionSize() { + return 10000000; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/54e39704/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index b66c331..76e3500 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -532,4 +532,25 @@ public class TestRegionServerMetrics { table.close(); } + + @Test + public void testAverageRegionSize() throws Exception { + TableName tableName = TableName.valueOf("testAverageRegionSize"); + byte[] cf = Bytes.toBytes("d"); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] val = Bytes.toBytes("Value"); + + //Force a hfile. + HTable t = TEST_UTIL.createTable(tableName, cf); + Put p = new Put(row); + p.add(cf, qualifier, val); + t.put(p); + TEST_UTIL.getHBaseAdmin().flush(tableName.getName()); + + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0); + + t.close(); + } }