Repository: hbase
Updated Branches:
  refs/heads/0.98 47c19607c -> e3ef8b69b
  refs/heads/branch-1 84dd9cbcb -> 8efc6148b


HBASE-16091 Canary takes lot more time when there are delete markers in the 
table (Vishal Khandelwal)

Conflicts:
        hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java

Amending-Author: Andrew Purtell <apurt...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8efc6148
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8efc6148
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8efc6148

Branch: refs/heads/branch-1
Commit: 8efc6148b9ccaa29d2608d1d7348d0d3c5d8158d
Parents: 84dd9cb
Author: Andrew Purtell <apurt...@apache.org>
Authored: Tue Jul 5 10:11:08 2016 -0700
Committer: Andrew Purtell <apurt...@apache.org>
Committed: Tue Jul 5 10:34:27 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HConstants.java     |  2 +
 .../org/apache/hadoop/hbase/tool/Canary.java    | 62 +++++++++++++++-----
 .../hadoop/hbase/tool/TestCanaryTool.java       | 29 +++++++--
 3 files changed, 73 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8efc6148/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e062989..769945b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1237,6 +1237,8 @@ public final class HConstants {
 
   public static final String HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY =
       "hbase.canary.write.table.check.period";
+  
+  public static final String HBASE_CANARY_READ_RAW_SCAN_KEY = 
"hbase.canary.read.raw.enabled";
 
   /**
    * Configuration keys for programmatic JAAS configuration for secured ZK 
interaction

http://git-wip-us.apache.org/repos/asf/hbase/blob/8efc6148/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index 7d37161..ca27e71 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -262,12 +262,15 @@ public final class Canary implements Tool {
     private HRegionInfo region;
     private Sink sink;
     private TaskType taskType;
+    private boolean rawScanEnabled;
 
-    RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType 
taskType) {
+    RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType 
taskType,
+        boolean rawScanEnabled) {
       this.connection = connection;
       this.region = region;
       this.sink = sink;
       this.taskType = taskType;
+      this.rawScanEnabled = rawScanEnabled;
     }
 
     @Override
@@ -321,6 +324,11 @@ public final class Canary implements Tool {
           get.addFamily(column.getName());
         } else {
           scan = new Scan();
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(String.format("rawScan : %s for table: %s", 
rawScanEnabled,
+              tableDesc.getTableName()));
+          }
+          scan.setRaw(rawScanEnabled);
           scan.setCaching(1);
           scan.setCacheBlocks(false);
           scan.setFilter(new FirstKeyOnlyFilter());
@@ -746,6 +754,8 @@ public final class Canary implements Tool {
     System.err.println("   -treatFailureAsError treats read / write failure as 
error");
     System.err.println("   -writeTable    The table used for write sniffing."
         + " Default is hbase:canary");
+    System.err.println("   -Dhbase.canary.read.raw.enabled=<true/false> Use 
this flag to enable or disable raw scan during read canary test"
+        + " Default is false and raw is not enabled during scan");
     System.err
         .println("   -D<configProperty>=<value> assigning or override the 
configuration params");
     System.exit(USAGE_EXIT_CODE);
@@ -870,6 +880,7 @@ public final class Canary implements Tool {
     private float regionsLowerLimit;
     private float regionsUpperLimit;
     private int checkPeriod;
+    private boolean rawScanEnabled;
 
     public RegionMonitor(Connection connection, String[] monitorTargets, 
boolean useRegExp,
         Sink sink, ExecutorService executor, boolean writeSniffing, TableName 
writeTableName,
@@ -887,6 +898,7 @@ public final class Canary implements Tool {
       this.checkPeriod =
           conf.getInt(HConstants.HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY,
             DEFAULT_WRITE_TABLE_CHECK_PERIOD);
+      this.rawScanEnabled = 
conf.getBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, false);
     }
 
     @Override
@@ -898,7 +910,8 @@ public final class Canary implements Tool {
             String[] tables = generateMonitorTables(this.targets);
             this.initialized = true;
             for (String table : tables) {
-              taskFutures.addAll(Canary.sniff(admin, sink, table, executor, 
TaskType.READ));
+              taskFutures.addAll(Canary.sniff(admin, sink, table, executor, 
TaskType.READ,
+                this.rawScanEnabled));
             }
           } else {
             taskFutures.addAll(sniff(TaskType.READ));
@@ -914,8 +927,8 @@ public final class Canary implements Tool {
               lastCheckTime = EnvironmentEdgeManager.currentTime();
             }
             // sniff canary table with write operation
-            taskFutures.addAll(Canary.sniff(admin, sink,
-              admin.getTableDescriptor(writeTableName), executor, 
TaskType.WRITE));
+            taskFutures.addAll(Canary.sniff(admin, sink, 
admin.getTableDescriptor(writeTableName),
+              executor, TaskType.WRITE, this.rawScanEnabled));
           }
 
           for (Future<Void> future : taskFutures) {
@@ -987,7 +1000,7 @@ public final class Canary implements Tool {
       for (HTableDescriptor table : admin.listTables()) {
         if (admin.isTableEnabled(table.getTableName())
             && (!table.getTableName().equals(writeTableName))) {
-          taskFutures.addAll(Canary.sniff(admin, sink, table, executor, 
taskType));
+          taskFutures.addAll(Canary.sniff(admin, sink, table, executor, 
taskType, this.rawScanEnabled));
         }
       }
       return taskFutures;
@@ -1058,38 +1071,58 @@ public final class Canary implements Tool {
    * Canary entry point for specified table.
    * @throws Exception
    */
+  public static void sniff(final Admin admin, TableName tableName, boolean 
rawScanEnabled)
+      throws Exception {
+    sniff(admin, tableName, TaskType.READ, rawScanEnabled);
+  }
+  
+  /**
+   * Canary entry point for specified table.
+   * Keeping this method backward compatibility
+   * @throws Exception
+   */
   public static void sniff(final Admin admin, TableName tableName)
       throws Exception {
-    sniff(admin, tableName, TaskType.READ);
+    sniff(admin, tableName, TaskType.READ, false);
   }
 
   /**
    * Canary entry point for specified table with task type(read/write)
    * @throws Exception
    */
-  public static void sniff(final Admin admin, TableName tableName, TaskType 
taskType)
-      throws Exception {
+  public static void sniff(final Admin admin, TableName tableName, TaskType 
taskType,
+      boolean rawScanEnabled)   throws Exception {
     List<Future<Void>> taskFutures =
         Canary.sniff(admin, new StdOutSink(), tableName.getNameAsString(),
-          new ScheduledThreadPoolExecutor(1), taskType);
+          new ScheduledThreadPoolExecutor(1), taskType, rawScanEnabled);
     for (Future<Void> future : taskFutures) {
       future.get();
     }
   }
+  
+  /**
+   * Canary entry point for specified table with task type(read/write)
+   * Keeping this method backward compatible
+   * @throws Exception
+   */
+  public static void sniff(final Admin admin, TableName tableName, TaskType 
taskType)
+      throws Exception {
+    Canary.sniff(admin, tableName, taskType, false);
+  }
 
   /**
    * Canary entry point for specified table.
    * @throws Exception
    */
   private static List<Future<Void>> sniff(final Admin admin, final Sink sink, 
String tableName,
-      ExecutorService executor, TaskType taskType) throws Exception {
+      ExecutorService executor, TaskType taskType, boolean rawScanEnabled) 
throws Exception {
     if (LOG.isDebugEnabled()) {
       LOG.debug(String.format("checking table is enabled and getting table 
descriptor for table %s",
         tableName));
     }
     if (admin.isTableEnabled(TableName.valueOf(tableName))) {
       return Canary.sniff(admin, sink, 
admin.getTableDescriptor(TableName.valueOf(tableName)),
-        executor, taskType);
+        executor, taskType, rawScanEnabled);
     } else {
       LOG.warn(String.format("Table %s is not enabled", tableName));
     }
@@ -1100,7 +1133,8 @@ public final class Canary implements Tool {
    * Loops over regions that owns this table, and output some information 
abouts the state.
    */
   private static List<Future<Void>> sniff(final Admin admin, final Sink sink,
-      HTableDescriptor tableDesc, ExecutorService executor, TaskType taskType) 
throws Exception {
+      HTableDescriptor tableDesc, ExecutorService executor, TaskType taskType,
+      boolean rawScanEnabled) throws Exception {
 
     if (LOG.isDebugEnabled()) {
       LOG.debug(String.format("reading list of regions for table %s", 
tableDesc.getTableName()));
@@ -1114,8 +1148,8 @@ public final class Canary implements Tool {
     }
     List<RegionTask> tasks = new ArrayList<RegionTask>();
     try {
-      for (HRegionInfo region : 
admin.getTableRegions(tableDesc.getTableName())) {
-        tasks.add(new RegionTask(admin.getConnection(), region, sink, 
taskType));
+      for (HRegionInfo region : 
admin.getTableRegions(tableDesc.getTableName())) {        
+        tasks.add(new RegionTask(admin.getConnection(), region, sink, 
taskType, rawScanEnabled));
       }
     } finally {
       table.close();

http://git-wip-us.apache.org/repos/asf/hbase/blob/8efc6148/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index 05852b8..b5b0d37 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -19,26 +19,21 @@
 
 package org.apache.hadoop.hbase.tool;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.log4j.Appender;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.spi.LoggingEvent;
 import com.google.common.collect.Iterables;
-import org.apache.hadoop.hbase.HConstants;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -148,6 +143,28 @@ public class TestCanaryTool {
     }));
   }
 
+  @Test
+  public void testRawScanConfig() throws Exception {
+    TableName tableName = TableName.valueOf("testTableRawScan");
+    Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY 
});
+    // insert some test rows
+    for (int i=0; i<1000; i++) {
+      byte[] iBytes = Bytes.toBytes(i);
+      Put p = new Put(iBytes);
+      p.addColumn(FAMILY, COLUMN, iBytes);
+      table.put(p);
+    }
+    ExecutorService executor = new ScheduledThreadPoolExecutor(1);
+    Canary.RegionServerStdOutSink sink = spy(new 
Canary.RegionServerStdOutSink());
+    Canary canary = new Canary(executor, sink);
+    String[] args = { "-t", "10000", "testTableRawScan" };
+    org.apache.hadoop.conf.Configuration conf = new 
org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
+    conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
+    ToolRunner.run(conf, canary, args);
+    verify(sink, atLeastOnce())
+        .publishReadTiming(isA(HRegionInfo.class), 
isA(HColumnDescriptor.class), anyLong());
+  }
+  
   private void runRegionserverCanary() throws Exception {
     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
     Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink());

Reply via email to