Repository: hbase Updated Branches: refs/heads/master 29c46c483 -> 318751cfd
HBASE-16091 Canary takes lot more time when there are delete markers in the table (Vishal Khandelwal) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/318751cf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/318751cf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/318751cf Branch: refs/heads/master Commit: 318751cfd621cfb848d90d623fdd9db1d19894ed Parents: 29c46c4 Author: Andrew Purtell <apurt...@apache.org> Authored: Tue Jul 5 10:11:08 2016 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Tue Jul 5 11:13:48 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/HConstants.java | 2 + .../org/apache/hadoop/hbase/tool/Canary.java | 63 +++++++++++++++----- .../hadoop/hbase/tool/TestCanaryTool.java | 22 +++++++ 3 files changed, 72 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/318751cf/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index fa4ce64..0bc0a07 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1244,6 +1244,8 @@ public final class HConstants { public static final String HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY = "hbase.canary.write.table.check.period"; + + public static final String HBASE_CANARY_READ_RAW_SCAN_KEY = "hbase.canary.read.raw.enabled"; /** * Configuration keys for programmatic JAAS configuration for secured ZK interaction http://git-wip-us.apache.org/repos/asf/hbase/blob/318751cf/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 360b0f5..2e7cf7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -264,12 +264,15 @@ public final class Canary implements Tool { private HRegionInfo region; private Sink sink; private TaskType taskType; + private boolean rawScanEnabled; - RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType taskType) { + RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType taskType, + boolean rawScanEnabled) { this.connection = connection; this.region = region; this.sink = sink; this.taskType = taskType; + this.rawScanEnabled = rawScanEnabled; } @Override @@ -323,7 +326,11 @@ public final class Canary implements Tool { get.addFamily(column.getName()); } else { scan = new Scan(); - scan.setRaw(true); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("rawScan : %s for table: %s", rawScanEnabled, + tableDesc.getTableName())); + } + scan.setRaw(rawScanEnabled); scan.setCaching(1); scan.setCacheBlocks(false); scan.setFilter(new FirstKeyOnlyFilter()); @@ -749,6 +756,8 @@ public final class Canary implements Tool { System.err.println(" -treatFailureAsError treats read / write failure as error"); System.err.println(" -writeTable The table used for write sniffing." + " Default is hbase:canary"); + System.err.println(" -Dhbase.canary.read.raw.enabled=<true/false> Use this flag to enable or disable raw scan during read canary test" + + " Default is false and raw is not enabled during scan"); System.err .println(" -D<configProperty>=<value> assigning or override the configuration params"); System.exit(USAGE_EXIT_CODE); @@ -873,6 +882,7 @@ public final class Canary implements Tool { private float regionsLowerLimit; private float regionsUpperLimit; private int checkPeriod; + private boolean rawScanEnabled; public RegionMonitor(Connection connection, String[] monitorTargets, boolean useRegExp, Sink sink, ExecutorService executor, boolean writeSniffing, TableName writeTableName, @@ -890,6 +900,7 @@ public final class Canary implements Tool { this.checkPeriod = conf.getInt(HConstants.HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY, DEFAULT_WRITE_TABLE_CHECK_PERIOD); + this.rawScanEnabled = conf.getBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, false); } @Override @@ -901,7 +912,8 @@ public final class Canary implements Tool { String[] tables = generateMonitorTables(this.targets); this.initialized = true; for (String table : tables) { - taskFutures.addAll(Canary.sniff(admin, sink, table, executor, TaskType.READ)); + taskFutures.addAll(Canary.sniff(admin, sink, table, executor, TaskType.READ, + this.rawScanEnabled)); } } else { taskFutures.addAll(sniff(TaskType.READ)); @@ -917,8 +929,8 @@ public final class Canary implements Tool { lastCheckTime = EnvironmentEdgeManager.currentTime(); } // sniff canary table with write operation - taskFutures.addAll(Canary.sniff(admin, sink, - admin.getTableDescriptor(writeTableName), executor, TaskType.WRITE)); + taskFutures.addAll(Canary.sniff(admin, sink, admin.getTableDescriptor(writeTableName), + executor, TaskType.WRITE, this.rawScanEnabled)); } for (Future<Void> future : taskFutures) { @@ -990,7 +1002,7 @@ public final class Canary implements Tool { for (HTableDescriptor table : admin.listTables()) { if (admin.isTableEnabled(table.getTableName()) && (!table.getTableName().equals(writeTableName))) { - taskFutures.addAll(Canary.sniff(admin, sink, table, executor, taskType)); + taskFutures.addAll(Canary.sniff(admin, sink, table, executor, taskType, this.rawScanEnabled)); } } return taskFutures; @@ -1056,38 +1068,58 @@ public final class Canary implements Tool { * Canary entry point for specified table. * @throws Exception */ + public static void sniff(final Admin admin, TableName tableName, boolean rawScanEnabled) + throws Exception { + sniff(admin, tableName, TaskType.READ, rawScanEnabled); + } + + /** + * Canary entry point for specified table. + * Keeping this method backward compatibility + * @throws Exception + */ public static void sniff(final Admin admin, TableName tableName) throws Exception { - sniff(admin, tableName, TaskType.READ); + sniff(admin, tableName, TaskType.READ, false); } /** * Canary entry point for specified table with task type(read/write) * @throws Exception */ - public static void sniff(final Admin admin, TableName tableName, TaskType taskType) - throws Exception { + public static void sniff(final Admin admin, TableName tableName, TaskType taskType, + boolean rawScanEnabled) throws Exception { List<Future<Void>> taskFutures = Canary.sniff(admin, new StdOutSink(), tableName.getNameAsString(), - new ScheduledThreadPoolExecutor(1), taskType); + new ScheduledThreadPoolExecutor(1), taskType, rawScanEnabled); for (Future<Void> future : taskFutures) { future.get(); } } + + /** + * Canary entry point for specified table with task type(read/write) + * Keeping this method backward compatible + * @throws Exception + */ + public static void sniff(final Admin admin, TableName tableName, TaskType taskType) + throws Exception { + Canary.sniff(admin, tableName, taskType, false); + } /** * Canary entry point for specified table. * @throws Exception */ private static List<Future<Void>> sniff(final Admin admin, final Sink sink, String tableName, - ExecutorService executor, TaskType taskType) throws Exception { + ExecutorService executor, TaskType taskType, boolean rawScanEnabled) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(String.format("checking table is enabled and getting table descriptor for table %s", tableName)); } if (admin.isTableEnabled(TableName.valueOf(tableName))) { return Canary.sniff(admin, sink, admin.getTableDescriptor(TableName.valueOf(tableName)), - executor, taskType); + executor, taskType, rawScanEnabled); } else { LOG.warn(String.format("Table %s is not enabled", tableName)); } @@ -1098,7 +1130,8 @@ public final class Canary implements Tool { * Loops over regions that owns this table, and output some information abouts the state. */ private static List<Future<Void>> sniff(final Admin admin, final Sink sink, - HTableDescriptor tableDesc, ExecutorService executor, TaskType taskType) throws Exception { + HTableDescriptor tableDesc, ExecutorService executor, TaskType taskType, + boolean rawScanEnabled) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(String.format("reading list of regions for table %s", tableDesc.getTableName())); @@ -1112,8 +1145,8 @@ public final class Canary implements Tool { } List<RegionTask> tasks = new ArrayList<RegionTask>(); try { - for (HRegionInfo region : admin.getTableRegions(tableDesc.getTableName())) { - tasks.add(new RegionTask(admin.getConnection(), region, sink, taskType)); + for (HRegionInfo region : admin.getTableRegions(tableDesc.getTableName())) { + tasks.add(new RegionTask(admin.getConnection(), region, sink, taskType, rawScanEnabled)); } } finally { table.close(); http://git-wip-us.apache.org/repos/asf/hbase/blob/318751cf/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java index fd67186..fb33d7e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java @@ -142,6 +142,28 @@ public class TestCanaryTool { })); } + @Test + public void testRawScanConfig() throws Exception { + TableName tableName = TableName.valueOf("testTableRawScan"); + Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); + // insert some test rows + for (int i=0; i<1000; i++) { + byte[] iBytes = Bytes.toBytes(i); + Put p = new Put(iBytes); + p.addColumn(FAMILY, COLUMN, iBytes); + table.put(p); + } + ExecutorService executor = new ScheduledThreadPoolExecutor(1); + Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink()); + Canary canary = new Canary(executor, sink); + String[] args = { "-t", "10000", "testTableRawScan" }; + org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration()); + conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true); + ToolRunner.run(conf, canary, args); + verify(sink, atLeastOnce()) + .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + } + private void runRegionserverCanary() throws Exception { ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink());
