Repository: hbase Updated Branches: refs/heads/master bdb46f01b -> b557f0bec
HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to single function. (Apekshit) Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b557f0be Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b557f0be Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b557f0be Branch: refs/heads/master Commit: b557f0bec62a48753e5d01d7a47f3c9e5a6b3ee8 Parents: bdb46f0 Author: Apekshit <apeksha...@gmail.com> Authored: Wed Jun 1 21:52:29 2016 -0700 Committer: Apekshit Sharma <a...@apache.org> Committed: Fri Jun 3 14:27:24 2016 -0700 ---------------------------------------------------------------------- .../regionserver/TestRegionServerMetrics.java | 682 +++++++------------ 1 file changed, 256 insertions(+), 426 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b557f0be/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 18796bd..06db468 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.regionserver; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; import static org.junit.Assert.*; @@ -39,23 +47,35 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - @Category({RegionServerTests.class, MediumTests.class}) public class TestRegionServerMetrics { - private static MetricsAssertHelper metricsHelper; + private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class); + + @Rule + public TestName testName = new TestName(); + + @ClassRule + public static TestRule timeout = CategoryBasedTimeout.forClass(TestRegionServerMetrics.class); static { Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); } + private static MetricsAssertHelper metricsHelper; private static MiniHBaseCluster cluster; private static HRegionServer rs; private static Configuration conf; private static HBaseTestingUtility TEST_UTIL; + private static Connection connection; private static MetricsRegionServer metricsRegionServer; private static MetricsRegionServerSource serverSource; private static final int NUM_SCAN_NEXT = 30; private static int numScanNext = 0; + private static byte[] cf = Bytes.toBytes("cf"); + private static byte[] row = Bytes.toBytes("row"); + private static byte[] qualifier = Bytes.toBytes("qual"); + private static byte[] val = Bytes.toBytes("val"); + private static Admin admin; @BeforeClass public static void startCluster() throws Exception { @@ -65,12 +85,16 @@ public class TestRegionServerMetrics { conf.getLong("hbase.splitlog.max.resubmit", 0); // Make the failure test faster conf.setInt("zookeeper.recovery.retry", 0); + // testMobMetrics creates few hfiles and manages compaction manually. + conf.setInt("hbase.hstore.compactionThreshold", 100); + conf.setInt("hbase.hstore.compaction.max", 100); conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); TEST_UTIL.startMiniCluster(1, 1); cluster = TEST_UTIL.getHBaseCluster(); - cluster.waitForActiveAndReadyMaster(); + admin = TEST_UTIL.getHBaseAdmin(); + connection = TEST_UTIL.getConnection(); while (cluster.getLiveRegionServerThreads().size() < 1) { Threads.sleep(100); @@ -88,551 +112,370 @@ public class TestRegionServerMetrics { } } - @Test(timeout = 300000) + TableName tableName; + Table table; + + @Before + public void beforeTestMethod() throws Exception { + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + tableName = TableName.valueOf(testName.getMethodName()); + table = TEST_UTIL.createTable(tableName, cf); + } + + @After + public void afterTestMethod() throws Exception { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + public void waitTableDeleted(TableName name, long timeoutInMillis) throws Exception { + long start = System.currentTimeMillis(); + while (true) { + HTableDescriptor[] tables = admin.listTables(); + for (HTableDescriptor htd : tables) { + if (htd.getNameAsString() == name.getNameAsString()) + return; + } + if (System.currentTimeMillis() - start > timeoutInMillis) + return; + Thread.sleep(1000); + } + } + + public void assertCounter(String metric, long expectedValue) { + metricsHelper.assertCounter(metric, expectedValue, serverSource); + } + + public void assertGauge(String metric, long expectedValue) { + metricsHelper.assertGauge(metric, expectedValue, serverSource); + } + + // Aggregates metrics from regions and assert given list of metrics and expected values. + public void assertRegionMetrics(String metric, long expectedValue) throws Exception { + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + for ( HRegionLocation location: locator.getAllRegionLocations()) { + HRegionInfo hri = location.getRegionInfo(); + MetricsRegionAggregateSource agg = + rs.getRegion(hri.getRegionName()).getMetrics().getSource().getAggregateSource(); + String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR + + "_table_" + tableName.getNameAsString() + + "_region_" + hri.getEncodedName()+ + "_metric_"; + metricsHelper.assertCounter(prefix + metric, expectedValue, agg); + } + } + } + + public void doNPuts(int n, boolean batch) throws Exception { + if (batch) { + List<Put> puts = new ArrayList<>(); + for (int i = 0; i < n; i++) { + Put p = new Put(Bytes.toBytes("" + i + "row")).addColumn(cf, qualifier, val); + puts.add(p); + } + table.put(puts); + } else { + for (int i = 0; i < n; i++) { + Put p = new Put(row).addColumn(cf, qualifier, val); + table.put(p); + } + } + } + + public void doNGets(int n, boolean batch) throws Exception { + if (batch) { + List<Get> gets = new ArrayList<>(); + for (int i = 0; i < n; i++) { + gets.add(new Get(row)); + } + table.get(gets); + } else { + for (int i = 0; i < n; i++) { + table.get(new Get(row)); + } + } + } + + @Test public void testRegionCount() throws Exception { - String regionMetricsKey = "regionCount"; - long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource); - // Creating a table should add one region - TEST_UTIL.createTable(TableName.valueOf("table"), Bytes.toBytes("cf")); - metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource); + metricsHelper.assertGauge("regionCount", 1, serverSource); } @Test public void testLocalFiles() throws Exception { - metricsHelper.assertGauge("percentFilesLocal", 0, serverSource); - metricsHelper.assertGauge("percentFilesLocalSecondaryRegions", 0, serverSource); + assertGauge("percentFilesLocal", 0); + assertGauge("percentFilesLocalSecondaryRegions", 0); } @Test public void testRequestCount() throws Exception { - String tableNameString = "testRequestCount"; - TableName tName = TableName.valueOf(tableNameString); - byte[] cfName = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] initValue = Bytes.toBytes("Value"); - - TEST_UTIL.createTable(tName, cfName); - - Connection connection = TEST_UTIL.getConnection(); - connection.getTable(tName).close(); //wait for the table to come up. - // Do a first put to be sure that the connection is established, meta is there and so on. - Table table = connection.getTable(tName); - Put p = new Put(row); - p.addColumn(cfName, qualifier, initValue); - table.put(p); + doNPuts(1, false); metricsRegionServer.getRegionServerWrapper().forceRecompute(); long requests = metricsHelper.getCounter("totalRequestCount", serverSource); long readRequests = metricsHelper.getCounter("readRequestCount", serverSource); long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource); - for (int i=0; i< 30; i++) { - table.put(p); - } + doNPuts(30, false); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource); - metricsHelper.assertCounter("readRequestCount", readRequests, serverSource); - metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); + assertCounter("totalRequestCount", requests + 30); + assertCounter("readRequestCount", readRequests); + assertCounter("writeRequestCount", writeRequests + 30); - Get g = new Get(row); - for (int i=0; i< 10; i++) { - table.get(g); - } + doNGets(10, false); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource); - metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource); - metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); + assertCounter("totalRequestCount", requests + 40); + assertCounter("readRequestCount", readRequests + 10); + assertCounter("writeRequestCount", writeRequests + 30); - try (RegionLocator locator = connection.getRegionLocator(tName)) { - for ( HRegionLocation location: locator.getAllRegionLocations()) { - HRegionInfo i = location.getRegionInfo(); - MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) - .getMetrics() - .getSource() - .getAggregateSource(); - String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ - "_table_"+tableNameString + - "_region_" + i.getEncodedName()+ - "_metric"; - metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg); - metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg); - } - } - List<Get> gets = new ArrayList<Get>(); - for (int i=0; i< 10; i++) { - gets.add(new Get(row)); - } - table.get(gets); + assertRegionMetrics("getNumOps", 10); + assertRegionMetrics("mutateCount", 31); + + doNGets(10, true); // true = batch metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource); - metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); - metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource); + assertCounter("totalRequestCount", requests + 50); + assertCounter("readRequestCount", readRequests + 20); + assertCounter("writeRequestCount", writeRequests + 30); - List<Put> puts = new ArrayList<>(); - for (int i=0; i< 30; i++) { - puts.add(p); - } - table.put(puts); + doNPuts(30, true); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource); - metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource); - metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource); - - table.close(); + assertCounter("totalRequestCount", requests + 80); + assertCounter("readRequestCount", readRequests + 20); + assertCounter("writeRequestCount", writeRequests + 60); } @Test public void testGet() throws Exception { - String tableNameString = "testGet"; - TableName tName = TableName.valueOf(tableNameString); - byte[] cfName = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] initValue = Bytes.toBytes("Value"); - - TEST_UTIL.createTable(tName, cfName); - - Connection connection = TEST_UTIL.getConnection(); - connection.getTable(tName).close(); //wait for the table to come up. - // Do a first put to be sure that the connection is established, meta is there and so on. - Table table = connection.getTable(tName); - Put p = new Put(row); - p.addColumn(cfName, qualifier, initValue); - table.put(p); - - Get g = new Get(row); - for (int i=0; i< 10; i++) { - table.get(g); - } - - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - - try (RegionLocator locator = connection.getRegionLocator(tName)) { - for ( HRegionLocation location: locator.getAllRegionLocations()) { - HRegionInfo i = location.getRegionInfo(); - MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) - .getMetrics() - .getSource() - .getAggregateSource(); - String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ - "_table_"+tableNameString + - "_region_" + i.getEncodedName()+ - "_metric"; - metricsHelper.assertCounter(prefix + "_getSizeNumOps", 10, agg); - metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg); - } - metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource); - } - table.close(); + doNPuts(1, false); + doNGets(10, false); + assertRegionMetrics("getNumOps", 10); + assertRegionMetrics("getSizeNumOps", 10); + metricsHelper.assertCounterGt("Get_num_ops", 10, serverSource); } @Test public void testMutationsWithoutWal() throws Exception { - TableName tableName = TableName.valueOf("testMutationsWithoutWal"); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("Value"); - - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - - Table t = TEST_UTIL.createTable(tableName, cf); - - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - p.setDurability(Durability.SKIP_WAL); - - t.put(p); + Put p = new Put(row).addColumn(cf, qualifier, val) + .setDurability(Durability.SKIP_WAL); + table.put(p); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource); + assertGauge("mutationsWithoutWALCount", 1); long minLength = row.length + cf.length + qualifier.length + val.length; metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource); - - t.close(); } @Test public void testStoreCount() throws Exception { - TableName tableName = TableName.valueOf("testStoreCount"); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("Value"); - - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - long stores = metricsHelper.getGaugeLong("storeCount", serverSource); - long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource); - //Force a hfile. - Table t = TEST_UTIL.createTable(tableName, cf); - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - t.put(p); + doNPuts(1, false); TEST_UTIL.getHBaseAdmin().flush(tableName); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertGauge("storeCount", stores +1, serverSource); - metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource); - - t.close(); + assertGauge("storeCount", 1); + assertGauge("storeFileCount", 1); } @Test public void testStoreFileAge() throws Exception { - TableName tableName = TableName.valueOf("testStoreFileAge"); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("Value"); - //Force a hfile. - Table t = TEST_UTIL.createTable(tableName, cf); - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - t.put(p); + doNPuts(1, false); TEST_UTIL.getHBaseAdmin().flush(tableName); metricsRegionServer.getRegionServerWrapper().forceRecompute(); assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0); assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0); assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0); - - t.close(); } @Test public void testCheckAndPutCount() throws Exception { - String tableNameString = "testCheckAndPutCount"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); byte[] valOne = Bytes.toBytes("Value"); byte[] valTwo = Bytes.toBytes("ValueTwo"); byte[] valThree = Bytes.toBytes("ValueThree"); - Table t = TEST_UTIL.createTable(tableName, cf); Put p = new Put(row); p.addColumn(cf, qualifier, valOne); - t.put(p); + table.put(p); Put pTwo = new Put(row); pTwo.addColumn(cf, qualifier, valTwo); - t.checkAndPut(row, cf, qualifier, valOne, pTwo); + table.checkAndPut(row, cf, qualifier, valOne, pTwo); Put pThree = new Put(row); pThree.addColumn(cf, qualifier, valThree); - t.checkAndPut(row, cf, qualifier, valOne, pThree); + table.checkAndPut(row, cf, qualifier, valOne, pThree); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource); - metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource); - - t.close(); + assertCounter("checkMutateFailedCount", 1); + assertCounter("checkMutatePassedCount", 1); } @Test public void testIncrement() throws Exception { - String tableNameString = "testIncrement"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes(0l); - - - Table t = TEST_UTIL.createTable(tableName, cf); - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - t.put(p); + Put p = new Put(row).addColumn(cf, qualifier, Bytes.toBytes(0L)); + table.put(p); - for(int count = 0; count< 13; count++) { + for(int count = 0; count < 13; count++) { Increment inc = new Increment(row); inc.addColumn(cf, qualifier, 100); - t.increment(inc); + table.increment(inc); } metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("incrementNumOps", 13, serverSource); - - t.close(); + assertCounter("incrementNumOps", 13); } @Test public void testAppend() throws Exception { - String tableNameString = "testAppend"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("One"); - - - Table t = TEST_UTIL.createTable(tableName, cf); - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - t.put(p); + doNPuts(1, false); for(int count = 0; count< 73; count++) { Append append = new Append(row); append.add(cf, qualifier, Bytes.toBytes(",Test")); - t.append(append); + table.append(append); } metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("appendNumOps", 73, serverSource); - - t.close(); + assertCounter("appendNumOps", 73); } @Test - public void testScanSize() throws IOException { - String tableNameString = "testScanSize"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("One"); - - List<Put> puts = new ArrayList<>(); - for (int insertCount =0; insertCount < 100; insertCount++) { - Put p = new Put(Bytes.toBytes("" + insertCount + "row")); - p.addColumn(cf, qualifier, val); - puts.add(p); - } - try (Table t = TEST_UTIL.createTable(tableName, cf)) { - t.put(puts); - - Scan s = new Scan(); - s.setBatch(1); - s.setCaching(1); - ResultScanner resultScanners = t.getScanner(s); - - for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { - Result result = resultScanners.next(); - assertNotNull(result); - assertEquals(1, result.size()); - } + public void testScanSize() throws Exception { + doNPuts(100, true); // batch put + Scan s = new Scan(); + s.setBatch(1); + s.setCaching(1); + ResultScanner resultScanners = table.getScanner(s); + + for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { + Result result = resultScanners.next(); + assertNotNull(result); + assertEquals(1, result.size()); } numScanNext += NUM_SCAN_NEXT; - try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { - for ( HRegionLocation location: locator.getAllRegionLocations()) { - HRegionInfo i = location.getRegionInfo(); - MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) - .getMetrics() - .getSource() - .getAggregateSource(); - String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ - "_table_"+tableNameString + - "_region_" + i.getEncodedName()+ - "_metric"; - metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg); - } - metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource); - } - try (Admin admin = TEST_UTIL.getHBaseAdmin()) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } + assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT); + assertCounter("ScanSize_num_ops", numScanNext); } @Test - public void testScanTime() throws IOException { - String tableNameString = "testScanTime"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("One"); - - List<Put> puts = new ArrayList<>(); - for (int insertCount =0; insertCount < 100; insertCount++) { - Put p = new Put(Bytes.toBytes("" + insertCount + "row")); - p.addColumn(cf, qualifier, val); - puts.add(p); - } - try (Table t = TEST_UTIL.createTable(tableName, cf)) { - t.put(puts); - - Scan s = new Scan(); - s.setBatch(1); - s.setCaching(1); - ResultScanner resultScanners = t.getScanner(s); - - for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { - Result result = resultScanners.next(); - assertNotNull(result); - assertEquals(1, result.size()); - } + public void testScanTime() throws Exception { + doNPuts(100, true); + Scan s = new Scan(); + s.setBatch(1); + s.setCaching(1); + ResultScanner resultScanners = table.getScanner(s); + + for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { + Result result = resultScanners.next(); + assertNotNull(result); + assertEquals(1, result.size()); } numScanNext += NUM_SCAN_NEXT; - try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { - for ( HRegionLocation location: locator.getAllRegionLocations()) { - HRegionInfo i = location.getRegionInfo(); - MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) - .getMetrics() - .getSource() - .getAggregateSource(); - String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ - "_table_"+tableNameString + - "_region_" + i.getEncodedName()+ - "_metric"; - metricsHelper.assertCounter(prefix + "_scanTimeNumOps", NUM_SCAN_NEXT, agg); - } - metricsHelper.assertCounter("ScanTime_num_ops", numScanNext, serverSource); - } - try (Admin admin = TEST_UTIL.getHBaseAdmin()) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } + assertRegionMetrics("scanTimeNumOps", NUM_SCAN_NEXT); + assertCounter("ScanTime_num_ops", numScanNext); } @Test - public void testScanSizeForSmallScan() throws IOException { - String tableNameString = "testScanSizeSmall"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("One"); - - List<Put> puts = new ArrayList<>(); - for (int insertCount =0; insertCount < 100; insertCount++) { - Put p = new Put(Bytes.toBytes("" + insertCount + "row")); - p.addColumn(cf, qualifier, val); - puts.add(p); - } - try (Table t = TEST_UTIL.createTable(tableName, cf)) { - t.put(puts); - - Scan s = new Scan(); - s.setSmall(true); - s.setCaching(1); - ResultScanner resultScanners = t.getScanner(s); - - for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { - Result result = resultScanners.next(); - assertNotNull(result); - assertEquals(1, result.size()); - } + public void testScanSizeForSmallScan() throws Exception { + doNPuts(100, true); + Scan s = new Scan(); + s.setSmall(true); + s.setCaching(1); + ResultScanner resultScanners = table.getScanner(s); + + for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { + Result result = resultScanners.next(); + assertNotNull(result); + assertEquals(1, result.size()); } numScanNext += NUM_SCAN_NEXT; - try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { - for ( HRegionLocation location: locator.getAllRegionLocations()) { - HRegionInfo i = location.getRegionInfo(); - MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName()) - .getMetrics() - .getSource() - .getAggregateSource(); - String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ - "_table_"+tableNameString + - "_region_" + i.getEncodedName()+ - "_metric"; - metricsHelper.assertCounter(prefix + "_scanSizeNumOps", NUM_SCAN_NEXT, agg); - } - metricsHelper.assertCounter("ScanSize_num_ops", numScanNext, serverSource); - } - try (Admin admin = TEST_UTIL.getHBaseAdmin()) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } + assertRegionMetrics("scanSizeNumOps", NUM_SCAN_NEXT); + assertCounter("ScanSize_num_ops", numScanNext); } @Test public void testMobMetrics() throws IOException, InterruptedException { - String tableNameString = "testMobMetrics"; - TableName tableName = TableName.valueOf(tableNameString); - byte[] cf = Bytes.toBytes("d"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("mobdata"); - int numHfiles = conf.getInt("hbase.hstore.compactionThreshold", 3) - 1; + TableName tableName = TableName.valueOf("testMobMetricsLocal"); + int numHfiles = 5; HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(cf); hcd.setMobEnabled(true); hcd.setMobThreshold(0); htd.addFamily(hcd); - Connection connection = ConnectionFactory.createConnection(conf); - Admin admin = connection.getAdmin(); - Table t = TEST_UTIL.createTable(htd, new byte[0][0], conf); - Region region = rs.getOnlineRegions(tableName).get(0); - for (int insertCount = 0; insertCount < numHfiles; insertCount++) { - Put p = new Put(Bytes.toBytes(insertCount)); - p.addColumn(cf, qualifier, val); - t.put(p); - admin.flush(tableName); - } - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("mobFlushCount", numHfiles, serverSource); - Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(2)); - ResultScanner scanner = t.getScanner(scan); - scanner.next(100); - numScanNext++; // this is an ugly construct - scanner.close(); - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("mobScanCellsCount", 2, serverSource); - region.getTableDesc().getFamily(cf).setMobThreshold(100); - ((HRegion)region).initialize(); - region.compact(true); - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - metricsHelper.assertCounter("cellsCountCompactedFromMob", numHfiles, - serverSource); - metricsHelper.assertCounter("cellsCountCompactedToMob", 0, serverSource); - scanner = t.getScanner(scan); - scanner.next(100); - numScanNext++; // this is an ugly construct - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - // metrics are reset by the region initialization - metricsHelper.assertCounter("mobScanCellsCount", 0, serverSource); - for (int insertCount = numHfiles; - insertCount < 2 * numHfiles - 1; insertCount++) { - Put p = new Put(Bytes.toBytes(insertCount)); - p.addColumn(cf, qualifier, val); - t.put(p); - admin.flush(tableName); + byte[] val = Bytes.toBytes("mobdata"); + try { + Table table = TEST_UTIL.createTable(htd, new byte[0][0], conf); + Region region = rs.getOnlineRegions(tableName).get(0); + for (int insertCount = 0; insertCount < numHfiles; insertCount++) { + Put p = new Put(Bytes.toBytes(insertCount)); + p.addColumn(cf, qualifier, val); + table.put(p); + admin.flush(tableName); + } + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertCounter("mobFlushCount", numHfiles); + + Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(numHfiles)); + ResultScanner scanner = table.getScanner(scan); + scanner.next(100); + numScanNext++; // this is an ugly construct + scanner.close(); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertCounter("mobScanCellsCount", numHfiles); + + region.getTableDesc().getFamily(cf).setMobThreshold(100); + // metrics are reset by the region initialization + ((HRegion) region).initialize(); + region.compact(true); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertCounter("cellsCountCompactedFromMob", numHfiles); + assertCounter("cellsCountCompactedToMob", 0); + + scanner = table.getScanner(scan); + scanner.next(100); + numScanNext++; // this is an ugly construct + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertCounter("mobScanCellsCount", 0); + + for (int insertCount = numHfiles; insertCount < 2 * numHfiles; insertCount++) { + Put p = new Put(Bytes.toBytes(insertCount)); + p.addColumn(cf, qualifier, val); + table.put(p); + admin.flush(tableName); + } + region.getTableDesc().getFamily(cf).setMobThreshold(0); + // metrics are reset by the region initialization + ((HRegion) region).initialize(); + region.compact(true); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + // metrics are reset by the region initialization + assertCounter("cellsCountCompactedFromMob", 0); + assertCounter("cellsCountCompactedToMob", 2 * numHfiles); + } finally { + admin.disableTable(tableName); + admin.deleteTable(tableName); } - region.getTableDesc().getFamily(cf).setMobThreshold(0); - ((HRegion)region).initialize(); - region.compact(true); - metricsRegionServer.getRegionServerWrapper().forceRecompute(); - // metrics are reset by the region initialization - metricsHelper.assertCounter("cellsCountCompactedFromMob", 0, serverSource); - metricsHelper.assertCounter("cellsCountCompactedToMob", 2 * numHfiles - 1, - serverSource); - t.close(); - admin.close(); - connection.close(); } @Test @Ignore public void testRangeCountMetrics() throws Exception { - String tableNameString = "testRangeCountMetrics"; final long[] timeranges = { 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 }; final String timeRangeType = "TimeRangeCount"; final String timeRangeMetricName = "Mutate"; boolean timeRangeCountUpdated = false; - TableName tName = TableName.valueOf(tableNameString); - byte[] cfName = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] initValue = Bytes.toBytes("Value"); - - TEST_UTIL.createTable(tName, cfName); - - Connection connection = TEST_UTIL.getConnection(); - connection.getTable(tName).close(); // wait for the table to come up. - // Do a first put to be sure that the connection is established, meta is there and so on. - Table table = connection.getTable(tName); Put p = new Put(row); - p.addColumn(cfName, qualifier, initValue); + p.addColumn(cf, qualifier, val); table.put(p); // do some puts and gets @@ -672,28 +515,15 @@ public class TestRegionServerMetrics { } } assertEquals(true, timeRangeCountUpdated); - - table.close(); } @Test public void testAverageRegionSize() throws Exception { - TableName tableName = TableName.valueOf("testAverageRegionSize"); - byte[] cf = Bytes.toBytes("d"); - byte[] row = Bytes.toBytes("rk"); - byte[] qualifier = Bytes.toBytes("qual"); - byte[] val = Bytes.toBytes("Value"); - //Force a hfile. - Table t = TEST_UTIL.createTable(tableName, cf); - Put p = new Put(row); - p.addColumn(cf, qualifier, val); - t.put(p); + doNPuts(1, false); TEST_UTIL.getHBaseAdmin().flush(tableName); metricsRegionServer.getRegionServerWrapper().forceRecompute(); assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0); - - t.close(); } }