[4/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)

[mbertozzi]

HBASE-14644 Region in transition metric is broken (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11844dca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11844dca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11844dca

Branch: refs/heads/branch-1.2
Commit: 11844dcaec01301dd3a3987d1375d40d0f83f439
Parents: d2d3dcd
Author: Matteo Bertozzi 
Authored: Fri Jun 10 21:49:10 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 10 21:57:21 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java  |   4 +
 .../org/apache/hadoop/hbase/master/HMaster.java |  25 +++-
 .../hbase/master/MetricsAssignmentManager.java  |   4 +
 .../hbase/regionserver/HRegionServer.java   |   4 -
 .../master/TestAssignmentManagerMetrics.java| 136 +++
 5 files changed, 167 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/11844dca/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 629d307..9308af6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -330,6 +330,10 @@ public class AssignmentManager extends ZooKeeperListener {
 useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
   }
 
+  MetricsAssignmentManager getAssignmentManagerMetrics() {
+return this.metricsAssignmentManager;
+  }
+
   /**
* Add the listener to the notification list.
* @param listener The AssignmentListener to register

http://git-wip-us.apache.org/repos/asf/hbase/blob/11844dca/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 5fff011..b373207 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.RegionStateListener;
+import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;
@@ -292,6 +293,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   private RegionNormalizerChore normalizerChore;
   private ClusterStatusChore clusterStatusChore;
   private ClusterStatusPublisher clusterStatusPublisherChore = null;
+  private PeriodicDoMetrics periodicDoMetricsChore = null;
 
   CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
@@ -343,6 +345,19 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
 }
   }
 
+  private static class PeriodicDoMetrics extends ScheduledChore {
+private final HMaster server;
+public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
+  super(server.getServerName() + "-DoMetricsChore", server, 
doMetricsInterval);
+  this.server = server;
+}
+
+@Override
+protected void chore() {
+  server.doMetrics();
+}
+  }
+
   /**
* Initializes the HMaster. The steps are as follows:
* 
@@ -410,6 +425,10 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   }
 }
 
+// Do Metrics periodically
+periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
+getChoreService().scheduleChore(periodicDoMetricsChore);
+
 // Some unit tests don't need a cluster, so no zookeeper at all
 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
   activeMasterManager = new ActiveMasterManager(zooKeeper, 
this.serverName, this);
@@ -535,8 +554,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
* Emit the HMaster metrics, such as region in transition metrics.
* Surrounding in a try block just to be sure metrics doesn't abort HMaster.
*/
-  @Override
-  protected void doMetrics() {
+  private void doMetrics() {
 try {
   if (assignmentManager != null) {
 assignmentManager.updateRegionsInTransitionMetrics();
@@ -1183,6 +1201,9 @@ 

[2/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)

[mbertozzi]

HBASE-14644 Region in transition metric is broken (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1bec9d0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1bec9d0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1bec9d0

Branch: refs/heads/branch-1
Commit: d1bec9d0b8f26543ff88299e7c456d678506332d
Parents: 37807b3
Author: Matteo Bertozzi 
Authored: Fri Jun 10 21:49:10 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 10 21:53:14 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java  |   4 +
 .../org/apache/hadoop/hbase/master/HMaster.java |  25 +++-
 .../hbase/master/MetricsAssignmentManager.java  |   4 +
 .../hbase/regionserver/HRegionServer.java   |   4 -
 .../master/TestAssignmentManagerMetrics.java| 136 +++
 5 files changed, 167 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d1bec9d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 7ae3e24..11e0798 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -331,6 +331,10 @@ public class AssignmentManager extends ZooKeeperListener {
 useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
   }
 
+  MetricsAssignmentManager getAssignmentManagerMetrics() {
+return this.metricsAssignmentManager;
+  }
+
   /**
* Add the listener to the notification list.
* @param listener The AssignmentListener to register

http://git-wip-us.apache.org/repos/asf/hbase/blob/d1bec9d0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 4b7e181..086329e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.RegionStateListener;
+import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;
@@ -304,6 +305,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   private RegionNormalizerChore normalizerChore;
   private ClusterStatusChore clusterStatusChore;
   private ClusterStatusPublisher clusterStatusPublisherChore = null;
+  private PeriodicDoMetrics periodicDoMetricsChore = null;
 
   CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
@@ -358,6 +360,19 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
 }
   }
 
+  private static class PeriodicDoMetrics extends ScheduledChore {
+private final HMaster server;
+public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
+  super(server.getServerName() + "-DoMetricsChore", server, 
doMetricsInterval);
+  this.server = server;
+}
+
+@Override
+protected void chore() {
+  server.doMetrics();
+}
+  }
+
   /**
* Initializes the HMaster. The steps are as follows:
* 
@@ -425,6 +440,10 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   }
 }
 
+// Do Metrics periodically
+periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
+getChoreService().scheduleChore(periodicDoMetricsChore);
+
 // Some unit tests don't need a cluster, so no zookeeper at all
 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
   activeMasterManager = new ActiveMasterManager(zooKeeper, 
this.serverName, this);
@@ -550,8 +569,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
* Emit the HMaster metrics, such as region in transition metrics.
* Surrounding in a try block just to be sure metrics doesn't abort HMaster.
*/
-  @Override
-  protected void doMetrics() {
+  private void doMetrics() {
 try {
   if (assignmentManager != null) {
 assignmentManager.updateRegionsInTransitionMetrics();
@@ -1209,6 +1227,9 @@ 

[1/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)

[mbertozzi]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 37807b38a -> d1bec9d0b
  refs/heads/branch-1.2 d2d3dcdae -> 11844dcae
  refs/heads/branch-1.3 d8d63d671 -> dac8c2bb1
  refs/heads/master bd45cf347 -> 4c885f4fd


HBASE-14644 Region in transition metric is broken (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c885f4f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c885f4f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c885f4f

Branch: refs/heads/master
Commit: 4c885f4fdbef1efa7f4b6f43bc024db6280cde39
Parents: bd45cf3
Author: Matteo Bertozzi 
Authored: Fri Jun 10 21:49:10 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 10 21:52:24 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java  |   4 +
 .../org/apache/hadoop/hbase/master/HMaster.java |  26 +++-
 .../hbase/master/MetricsAssignmentManager.java  |   4 +
 .../hbase/regionserver/HRegionServer.java   |   4 -
 .../master/TestAssignmentManagerMetrics.java| 136 +++
 5 files changed, 168 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4c885f4f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index f16463f..49026ac 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -258,6 +258,10 @@ public class AssignmentManager {
 this.tableLockManager = tableLockManager;
   }
 
+  MetricsAssignmentManager getAssignmentManagerMetrics() {
+return this.metricsAssignmentManager;
+  }
+
   /**
* Add the listener to the notification list.
* @param listener The AssignmentListener to register

http://git-wip-us.apache.org/repos/asf/hbase/blob/4c885f4f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d368ffb..efb6b6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.RegionStateListener;
+import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;
@@ -305,6 +306,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
   private RegionNormalizerChore normalizerChore;
   private ClusterStatusChore clusterStatusChore;
   private ClusterStatusPublisher clusterStatusPublisherChore = null;
+  private PeriodicDoMetrics periodicDoMetricsChore = null;
 
   CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
@@ -370,6 +372,19 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 }
   }
 
+  private static class PeriodicDoMetrics extends ScheduledChore {
+private final HMaster server;
+public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
+  super(server.getServerName() + "-DoMetricsChore", server, 
doMetricsInterval);
+  this.server = server;
+}
+
+@Override
+protected void chore() {
+  server.doMetrics();
+}
+  }
+
   /**
* Initializes the HMaster. The steps are as follows:
* 
@@ -433,6 +448,10 @@ public class HMaster extends HRegionServer implements 
MasterServices {
   }
 }
 
+// Do Metrics periodically
+periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
+getChoreService().scheduleChore(periodicDoMetricsChore);
+
 // Some unit tests don't need a cluster, so no zookeeper at all
 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
   activeMasterManager = new ActiveMasterManager(zooKeeper, 
this.serverName, this);
@@ -563,8 +582,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
* Emit the HMaster metrics, such as region in transition metrics.
* Surrounding in a try block just to be sure metrics doesn't abort HMaster.
*/
-  @Override
-  protected void doMetrics() {
+  private void doMetrics() {
  

[3/4] hbase git commit: HBASE-14644 Region in transition metric is broken (Huaxiang Sun)

[mbertozzi]

HBASE-14644 Region in transition metric is broken (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dac8c2bb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dac8c2bb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dac8c2bb

Branch: refs/heads/branch-1.3
Commit: dac8c2bb1017b77633afbef1430b08ac82d99ce4
Parents: d8d63d6
Author: Matteo Bertozzi 
Authored: Fri Jun 10 21:49:10 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 10 21:55:37 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java  |   4 +
 .../org/apache/hadoop/hbase/master/HMaster.java |  25 +++-
 .../hbase/master/MetricsAssignmentManager.java  |   4 +
 .../hbase/regionserver/HRegionServer.java   |   4 -
 .../master/TestAssignmentManagerMetrics.java| 136 +++
 5 files changed, 167 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dac8c2bb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 6223c65..4ed162f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -331,6 +331,10 @@ public class AssignmentManager extends ZooKeeperListener {
 useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
   }
 
+  MetricsAssignmentManager getAssignmentManagerMetrics() {
+return this.metricsAssignmentManager;
+  }
+
   /**
* Add the listener to the notification list.
* @param listener The AssignmentListener to register

http://git-wip-us.apache.org/repos/asf/hbase/blob/dac8c2bb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9323c36..f9d68f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.PleaseHoldException;
 import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.RegionStateListener;
+import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;
@@ -304,6 +305,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   private RegionNormalizerChore normalizerChore;
   private ClusterStatusChore clusterStatusChore;
   private ClusterStatusPublisher clusterStatusPublisherChore = null;
+  private PeriodicDoMetrics periodicDoMetricsChore = null;
 
   CatalogJanitor catalogJanitorChore;
   private LogCleaner logCleaner;
@@ -358,6 +360,19 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
 }
   }
 
+  private static class PeriodicDoMetrics extends ScheduledChore {
+private final HMaster server;
+public PeriodicDoMetrics(int doMetricsInterval, final HMaster server) {
+  super(server.getServerName() + "-DoMetricsChore", server, 
doMetricsInterval);
+  this.server = server;
+}
+
+@Override
+protected void chore() {
+  server.doMetrics();
+}
+  }
+
   /**
* Initializes the HMaster. The steps are as follows:
* 
@@ -425,6 +440,10 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   }
 }
 
+// Do Metrics periodically
+periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
+getChoreService().scheduleChore(periodicDoMetricsChore);
+
 // Some unit tests don't need a cluster, so no zookeeper at all
 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
   activeMasterManager = new ActiveMasterManager(zooKeeper, 
this.serverName, this);
@@ -550,8 +569,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
* Emit the HMaster metrics, such as region in transition metrics.
* Surrounding in a try block just to be sure metrics doesn't abort HMaster.
*/
-  @Override
-  protected void doMetrics() {
+  private void doMetrics() {
 try {
   if (assignmentManager != null) {
 assignmentManager.updateRegionsInTransitionMetrics();
@@ -1209,6 +1227,9 @@ 

[40/50] hbase git commit: HBASE-15959 Fix flaky test TestRegionServerMetrics.testMobMetrics (Huaxiang Sun and Jingcheng Du)

[syuanjiang]

HBASE-15959 Fix flaky test TestRegionServerMetrics.testMobMetrics (Huaxiang Sun 
and Jingcheng Du)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7172d56
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7172d56
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7172d56

Branch: refs/heads/hbase-12439
Commit: a7172d5611f33dde2566a4e140aeddc675a56f3a
Parents: d9463bc
Author: Jonathan M Hsieh 
Authored: Wed Jun 8 19:07:13 2016 -0700
Committer: Jonathan M Hsieh 
Committed: Wed Jun 8 19:07:13 2016 -0700

--
 .../hadoop/hbase/regionserver/TestRegionServerMetrics.java   | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a7172d56/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 06db468..89a82a7 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -451,6 +451,10 @@ public class TestRegionServerMetrics {
 admin.flush(tableName);
   }
   region.getTableDesc().getFamily(cf).setMobThreshold(0);
+
+  // closing the region forces the compaction.discharger to archive the 
compacted hfiles
+  ((HRegion) region).close();
+
   // metrics are reset by the region initialization
   ((HRegion) region).initialize();
   region.compact(true);

[43/50] hbase git commit: HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Adds a FifoRpcSchedulerFactory so you can try Fifo scheduler.

[syuanjiang]

HBASE-15994 Allow selection of RpcSchedulers
Adds logging by the RpcExecutors of their run configs
Adds a FifoRpcSchedulerFactory so you can try Fifo scheduler.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/031b7450
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/031b7450
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/031b7450

Branch: refs/heads/hbase-12439
Commit: 031b745001c7d54ef13f3cd6d725d0eb78095785
Parents: 407aa4d
Author: stack 
Authored: Wed Jun 8 20:23:11 2016 -0700
Committer: stack 
Committed: Wed Jun 8 20:23:11 2016 -0700

--
 .../hbase/ipc/BalancedQueueRpcExecutor.java |  4 ++
 .../hadoop/hbase/ipc/FifoRpcScheduler.java  |  5 ++
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 ---
 .../regionserver/FifoRpcSchedulerFactory.java   | 47 +
 .../hbase/regionserver/RpcSchedulerFactory.java |  4 +-
 .../regionserver/SimpleRpcSchedulerFactory.java |  6 +-
 .../regionserver/TestRpcSchedulerFactory.java   | 71 
 7 files changed, 161 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
index e4205eb..3505221 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
@@ -22,6 +22,8 @@ import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
@@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.util.ReflectionUtils;
 @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, 
HBaseInterfaceAudience.PHOENIX })
 @InterfaceStability.Evolving
 public class BalancedQueueRpcExecutor extends RpcExecutor {
+  private static final Log LOG = 
LogFactory.getLog(BalancedQueueRpcExecutor.class);
 
   protected final List queues;
   private final QueueBalancer balancer;
@@ -62,6 +65,7 @@ public class BalancedQueueRpcExecutor extends RpcExecutor {
 queues = new ArrayList(numQueues);
 this.balancer = getBalancer(numQueues);
 initializeQueues(numQueues, queueClass, initargs);
+LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount);
   }
 
   protected void initializeQueues(final int numQueues,

http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
index ee36f3f..70d903a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DaemonThreadFactory;
 
@@ -32,6 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger;
  * This can be used for HMaster, where no prioritization is needed.
  */
 public class FifoRpcScheduler extends RpcScheduler {
+  private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class);
   private final int handlerCount;
   private final int maxQueueLength;
   private final AtomicInteger queueSize = new AtomicInteger(0);
@@ -41,6 +44,8 @@ public class FifoRpcScheduler extends RpcScheduler {
 this.handlerCount = handlerCount;
 this.maxQueueLength = 
conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH,
 handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
+LOG.info("Using " + this.getClass().getSimpleName() + " as user call 
queue; handlerCount=" +
+handlerCount + "; maxQueueLength=" + maxQueueLength);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/031b7450/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcScheduler.java

[48/50] hbase git commit: HBASE-15958 Implement ClaimQueues on top of HBase

[syuanjiang]

HBASE-15958 Implement ClaimQueues on top of HBase

Building on HBase-15883.
Now implementing the claim queues procedure within an HBase table.
Also added UnitTests to test claimQueue.
Peer tracking will still be performed by ZooKeeper though.
Also modified the queueId tracking procedure so we no longer have to perform 
scans over the Replication Table.
This does make our queue naming schema slightly different from 
ReplicationQueuesZKImpl though.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/babdedc1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/babdedc1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/babdedc1

Branch: refs/heads/hbase-12439
Commit: babdedc1b0f0159eb526fb5c9ee08525de7ce404
Parents: 108d39a
Author: Joseph Hwang 
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 9 15:05:54 2016 -0700

--
 .../hbase/replication/ReplicationQueues.java|   8 +-
 .../replication/ReplicationQueuesArguments.java |   4 +-
 .../replication/ReplicationQueuesHBaseImpl.java | 485 ---
 .../replication/ReplicationQueuesZKImpl.java|  26 +-
 .../regionserver/ReplicationSourceManager.java  |   7 +-
 .../replication/TestReplicationStateBasic.java  |   6 +-
 .../TestReplicationStateHBaseImpl.java  | 302 +---
 .../replication/TestReplicationStateZKImpl.java |   1 -
 .../TestReplicationSourceManager.java   |  12 +-
 9 files changed, 579 insertions(+), 272 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 809b122..0de0cc8 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -19,8 +19,8 @@
 package org.apache.hadoop.hbase.replication;
 
 import java.util.List;
-import java.util.SortedMap;
-import java.util.SortedSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 
@@ -96,10 +96,10 @@ public interface ReplicationQueues {
   /**
* Take ownership for the set of queues belonging to a dead region server.
* @param regionserver the id of the dead region server
-   * @return A SortedMap of the queues that have been claimed, including a 
SortedSet of WALs in
+   * @return A Map of the queues that have been claimed, including a Set of 
WALs in
* each queue. Returns an empty map if no queues were failed-over.
*/
-  SortedMap claimQueues(String regionserver);
+  Map claimQueues(String regionserver);
 
   /**
* Get a list of all region servers that have outstanding replication 
queues. These servers could

http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
index 4907b73..4fdc4e7 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
@@ -56,11 +56,11 @@ public class ReplicationQueuesArguments {
 this.conf = conf;
   }
 
-  public Abortable getAbort() {
+  public Abortable getAbortable() {
 return abort;
   }
 
-  public void setAbort(Abortable abort) {
+  public void setAbortable(Abortable abort) {
 this.abort = abort;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
index 29f0632..34a5289 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
+++ 

[22/50] hbase git commit: HBASE-15883 Adding WAL files and tracking offsets in HBase.

[syuanjiang]

HBASE-15883 Adding WAL files and tracking offsets in HBase.

Implemented ReplicationQueuesHBaseImpl that tracks WAL offsets and replication 
queues in an HBase table.
Only wrote the basic tracking methods, have not implemented claimQueue() or 
HFileRef methods yet.
Wrote a basic unit test for ReplicationQueueHBaseImpl that tests the 
implemented functions on a single Region Server

Signed-off-by: Elliott Clark 
Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21e98271
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21e98271
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21e98271

Branch: refs/heads/hbase-12439
Commit: 21e98271c32f0d44106515a72b2c92d518c03668
Parents: 9a53d8b
Author: Joseph Hwang 
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 3 15:23:10 2016 -0700

--
 .../hbase/replication/ReplicationFactory.java   |  11 +-
 .../hbase/replication/ReplicationQueues.java|   8 +-
 .../replication/ReplicationQueuesArguments.java |  66 +++
 .../replication/ReplicationQueuesHBaseImpl.java | 491 +++
 .../replication/ReplicationQueuesZKImpl.java|  13 +-
 .../replication/regionserver/Replication.java   |  12 +-
 .../regionserver/ReplicationSourceManager.java  |   5 +-
 .../replication/TestReplicationAdmin.java   |   3 +-
 .../hbase/master/cleaner/TestLogsCleaner.java   |   3 +-
 .../cleaner/TestReplicationHFileCleaner.java|   4 +-
 .../replication/TestReplicationStateBasic.java  |   2 +-
 .../TestReplicationStateHBaseImpl.java  | 243 +
 .../replication/TestReplicationStateZKImpl.java |  13 +-
 .../TestReplicationSourceManager.java   |  36 +-
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |   4 +-
 15 files changed, 871 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 91e77ca..e264a4d 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import org.apache.commons.lang.reflect.ConstructorUtils;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
@@ -30,9 +31,11 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 @InterfaceAudience.Private
 public class ReplicationFactory {
 
-  public static ReplicationQueues getReplicationQueues(final ZooKeeperWatcher 
zk,
-  Configuration conf, Abortable abortable) {
-return new ReplicationQueuesZKImpl(zk, conf, abortable);
+  public static ReplicationQueues 
getReplicationQueues(ReplicationQueuesArguments args)
+  throws Exception {
+Class classToBuild = args.getConf().getClass("hbase.region.replica." +
+"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class);
+return (ReplicationQueues) 
ConstructorUtils.invokeConstructor(classToBuild, args);
   }
 
   public static ReplicationQueuesClient getReplicationQueuesClient(final 
ZooKeeperWatcher zk,
@@ -44,7 +47,7 @@ public class ReplicationFactory {
   Abortable abortable) {
 return getReplicationPeers(zk, conf, null, abortable);
   }
-  
+
   public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher 
zk, Configuration conf,
   final ReplicationQueuesClient queuesClient, Abortable abortable) {
 return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);

http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0d47a88..db6da91 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -83,13 +83,13 @@ public interface ReplicationQueues {
   /**
* Get a list of all WALs in the given queue.
* @param queueId a String that identifies the 

[47/50] hbase git commit: HBASE-15925 provide default values for hadoop compat module related properties that match default hadoop profile.

[syuanjiang]

HBASE-15925 provide default values for hadoop compat module related properties 
that match default hadoop profile.

Signed-off-by: Mikhail Antonov 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/108d39a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/108d39a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/108d39a7

Branch: refs/heads/hbase-12439
Commit: 108d39a7277bdcbc8b4bc9c50bab457a86a71971
Parents: 55a04b7
Author: Sean Busbey 
Authored: Thu Jun 9 13:30:45 2016 -0500
Committer: Sean Busbey 
Committed: Thu Jun 9 16:29:28 2016 -0500

--
 pom.xml | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/108d39a7/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6652b78..0e33ae8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1212,6 +1212,13 @@
 
 2.7.1
 3.0.0-SNAPSHOT
+
+${hadoop-two.version}
+hbase-hadoop2-compat
+src/main/assembly/hadoop-two-compat.xml
+
 1.2
 1.9
 

[44/50] hbase git commit: HBASE-15975 logic in TestHTableDescriptor#testAddCoprocessorWithSpecStr is wrong (Huaxiang Sun)

[syuanjiang]

HBASE-15975 logic in TestHTableDescriptor#testAddCoprocessorWithSpecStr is 
wrong (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41cc2155
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41cc2155
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41cc2155

Branch: refs/heads/hbase-12439
Commit: 41cc215544deb5f1f6cf522efd19229da0c99dd2
Parents: 031b745
Author: Matteo Bertozzi 
Authored: Wed Jun 8 22:42:07 2016 -0700
Committer: Matteo Bertozzi 
Committed: Wed Jun 8 22:42:50 2016 -0700

--
 .../hadoop/hbase/TestHTableDescriptor.java   | 19 ++-
 1 file changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/41cc2155/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
index 680f2c1..70380e6 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
@@ -55,35 +55,36 @@ public class TestHTableDescriptor {
   public void testAddCoprocessorWithSpecStr() throws IOException {
 HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
 String cpName = "a.b.c.d";
-boolean expected = false;
 try {
   htd.addCoprocessorWithSpec(cpName);
+  fail();
 } catch (IllegalArgumentException iae) {
-  expected = true;
+  // Expected as cpName is invalid
 }
-if (!expected) fail();
+
 // Try minimal spec.
 try {
   htd.addCoprocessorWithSpec("file:///some/path" + "|" + cpName);
+  fail();
 } catch (IllegalArgumentException iae) {
-  expected = false;
+  // Expected to be invalid
 }
-if (expected) fail();
+
 // Try more spec.
 String spec = 
"hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2";
 try {
   htd.addCoprocessorWithSpec(spec);
 } catch (IllegalArgumentException iae) {
-  expected = false;
+  fail();
 }
-if (expected) fail();
+
 // Try double add of same coprocessor
 try {
   htd.addCoprocessorWithSpec(spec);
+  fail();
 } catch (IOException ioe) {
-  expected = true;
+  // Expect that the coprocessor already exists
 }
-if (!expected) fail();
   }
 
   @Test

[41/50] hbase git commit: HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFactory.clas

[syuanjiang]

HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors 
of their run configs Changes the default RpcSchedulerFactory from 
SimpleRpcSchedulerFactory.class to RpcSchedulerFactoryImpl.class. 
RpcSchedulerFactoryImpl.class. defaults to using 
SimpleRpcSchedulerFactory.class and the SimpleRpcScheduler, as has been default 
up to this, unless you set "hbase.region.server.rpc.scheduler.class" to 
org.apache.hadoop.hbase.ipc.FifoRpcScheduler

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3ac4a57f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3ac4a57f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3ac4a57f

Branch: refs/heads/hbase-12439
Commit: 3ac4a57fd205e1909c874cabd7fda9fb176f3f0f
Parents: a7172d5
Author: stack 
Authored: Wed Jun 8 09:53:51 2016 -0700
Committer: stack 
Committed: Wed Jun 8 20:20:50 2016 -0700

--
 .../hbase/ipc/BalancedQueueRpcExecutor.java |  4 ++
 .../hadoop/hbase/ipc/FifoRpcScheduler.java  |  5 ++
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 ---
 .../regionserver/FifoRpcSchedulerFactory.java   | 47 +
 .../hbase/regionserver/RpcSchedulerFactory.java |  4 +-
 .../regionserver/SimpleRpcSchedulerFactory.java |  6 +-
 .../regionserver/TestRpcSchedulerFactory.java   | 71 
 7 files changed, 161 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3ac4a57f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
index e4205eb..3505221 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
@@ -22,6 +22,8 @@ import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
@@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.util.ReflectionUtils;
 @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, 
HBaseInterfaceAudience.PHOENIX })
 @InterfaceStability.Evolving
 public class BalancedQueueRpcExecutor extends RpcExecutor {
+  private static final Log LOG = 
LogFactory.getLog(BalancedQueueRpcExecutor.class);
 
   protected final List queues;
   private final QueueBalancer balancer;
@@ -62,6 +65,7 @@ public class BalancedQueueRpcExecutor extends RpcExecutor {
 queues = new ArrayList(numQueues);
 this.balancer = getBalancer(numQueues);
 initializeQueues(numQueues, queueClass, initargs);
+LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount);
   }
 
   protected void initializeQueues(final int numQueues,

http://git-wip-us.apache.org/repos/asf/hbase/blob/3ac4a57f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
index ee36f3f..70d903a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DaemonThreadFactory;
 
@@ -32,6 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger;
  * This can be used for HMaster, where no prioritization is needed.
  */
 public class FifoRpcScheduler extends RpcScheduler {
+  private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class);
   private final int handlerCount;
   private final int maxQueueLength;
   private final AtomicInteger queueSize = new AtomicInteger(0);
@@ -41,6 +44,8 @@ public class FifoRpcScheduler extends RpcScheduler {
 this.handlerCount = handlerCount;
 this.maxQueueLength = 
conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH,
 handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
+LOG.info("Using " + 

[35/50] hbase git commit: Revert "HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns connections to readers"" Revert mistaken commit... This reverts commit e0b70c00e74aeaac33570508e3732a

[syuanjiang]

Revert "HBASE-15948 Port "HADOOP-9956 RPC listener inefficiently assigns 
connections to readers""
Revert mistaken commit...
This reverts commit e0b70c00e74aeaac33570508e3732a53daea839e.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e66ecd7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e66ecd7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e66ecd7d

Branch: refs/heads/hbase-12439
Commit: e66ecd7db68d6ef57084543d08f7774c82f22f45
Parents: 6d5a259
Author: stack 
Authored: Tue Jun 7 16:41:30 2016 -0700
Committer: stack 
Committed: Tue Jun 7 16:41:30 2016 -0700

--
 .../hbase/ipc/MetricsHBaseServerSource.java |  10 +-
 .../ipc/MetricsHBaseServerWrapperImpl.java  |   6 +-
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  | 408 ---
 .../regionserver/SimpleRpcSchedulerFactory.java |   2 +-
 .../hadoop/hbase/ipc/AbstractTestIPC.java   |   2 +-
 5 files changed, 187 insertions(+), 241 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
index ce57e0f..bb89789 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
@@ -52,16 +52,14 @@ public interface MetricsHBaseServerSource extends 
BaseSource {
   String TOTAL_CALL_TIME_NAME = "totalCallTime";
   String TOTAL_CALL_TIME_DESC = "Total call time, including both queued and 
processing time.";
   String QUEUE_SIZE_NAME = "queueSize";
-  String QUEUE_SIZE_DESC = "Number of bytes in the call queues; request has 
been read and " +
-"parsed and is waiting to run or is currently being executed.";
+  String QUEUE_SIZE_DESC = "Number of bytes in the call queues.";
   String GENERAL_QUEUE_NAME = "numCallsInGeneralQueue";
-  String GENERAL_QUEUE_DESC = "Number of calls in the general call queue; " +
-"parsed requests waiting in scheduler to be executed";
+  String GENERAL_QUEUE_DESC = "Number of calls in the general call queue.";
   String PRIORITY_QUEUE_NAME = "numCallsInPriorityQueue";
   String REPLICATION_QUEUE_NAME = "numCallsInReplicationQueue";
   String REPLICATION_QUEUE_DESC =
-  "Number of calls in the replication call queue waiting to be run";
-  String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue 
waiting to be run";
+  "Number of calls in the replication call queue.";
+  String PRIORITY_QUEUE_DESC = "Number of calls in the priority call queue.";
   String NUM_OPEN_CONNECTIONS_NAME = "numOpenConnections";
   String NUM_OPEN_CONNECTIONS_DESC = "Number of open connections.";
   String NUM_ACTIVE_HANDLER_NAME = "numActiveHandler";

http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
index 4f53709..9979c75 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java
@@ -36,7 +36,7 @@ public class MetricsHBaseServerWrapperImpl implements 
MetricsHBaseServerWrapper
 if (!isServerStarted()) {
   return 0;
 }
-return server.callQueueSizeInBytes.get();
+return server.callQueueSize.get();
   }
 
   @Override
@@ -65,10 +65,10 @@ public class MetricsHBaseServerWrapperImpl implements 
MetricsHBaseServerWrapper
 
   @Override
   public int getNumOpenConnections() {
-if (!isServerStarted()) {
+if (!isServerStarted() || this.server.connectionList == null) {
   return 0;
 }
-return server.getNumOpenConnections();
+return server.connectionList.size();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/e66ecd7d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index aca3fdd..483ce86 100644
--- 

[50/50] hbase git commit: HBASE-16004 Update to Netty 4.1.1

[syuanjiang]

HBASE-16004 Update to Netty 4.1.1

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd45cf34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd45cf34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd45cf34

Branch: refs/heads/hbase-12439
Commit: bd45cf34762332a3a51f605798a3e050e7a1e62e
Parents: 6da6bab
Author: Jurriaan Mous 
Authored: Fri Jun 10 17:57:42 2016 +0200
Committer: stack 
Committed: Fri Jun 10 12:14:05 2016 -0700

--
 .../main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java| 4 ++--
 pom.xml  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index c1ed748..723a234 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -347,13 +347,13 @@ public class AsyncRpcClient extends AbstractRpcClient {
 }
 // do not close global EventLoopGroup.
 if (!useGlobalEventLoopGroup) {
-  bootstrap.group().shutdownGracefully();
+  bootstrap.config().group().shutdownGracefully();
 }
   }
 
   @Override
   public EventLoop getEventExecutor() {
-return this.bootstrap.group().next();
+return this.bootstrap.config().group().next();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 0e33ae8..93b7a0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1254,7 +1254,7 @@
 4.0.3
 2.4.1
 1.3.3
-4.0.30.Final
+4.1.1.Final
 3.6.2.Final
 2.1.2
 1.0.8

[33/50] hbase git commit: HBASE-15981 Remove references to disabling table in docs around stripe and date-tiered compactions

[syuanjiang]

HBASE-15981 Remove references to disabling table in docs around stripe and 
date-tiered compactions

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2da090f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2da090f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2da090f9

Branch: refs/heads/hbase-12439
Commit: 2da090f9a3ca42b07f342035c952ce0465c67265
Parents: 1125215
Author: Bryan Beaudreault 
Authored: Tue Jun 7 15:31:54 2016 -0400
Committer: stack 
Committed: Tue Jun 7 13:10:35 2016 -0700

--
 src/main/asciidoc/_chapters/architecture.adoc | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2da090f9/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc 
b/src/main/asciidoc/_chapters/architecture.adoc
index faa1230..9f59cd5 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -2119,7 +2119,7 @@ This is not necessary on new tables.
 [[ops.date.tiered.config]]
 == Configuring Date Tiered Compaction
 
-Each of the settings for date tiered compaction should be configured at the 
table or column family, after disabling the table.
+Each of the settings for date tiered compaction should be configured at the 
table or column family level.
 If you use HBase shell, the general command pattern is as follows:
 
 [source,sql]
@@ -2199,7 +2199,6 @@ You can enable stripe compaction for a table or a column 
family, by setting its
 You also need to set the `hbase.hstore.blockingStoreFiles` to a high number, 
such as 100 (rather than the default value of 10).
 
 .Procedure: Enable Stripe Compaction
-. If the table already exists, disable the table.
 . Run one of following commands in the HBase shell.
   Replace the table name `orders_table` with the name of your table.
 +
@@ -2215,7 +2214,6 @@ create 'orders_table', 'blobs_cf', CONFIGURATION => 
{'hbase.hstore.engine.class'
 . Enable the table.
 
 .Procedure: Disable Stripe Compaction
-. Disable the table.
 . Set the `hbase.hstore.engine.class` option to either nil or 
`org.apache.hadoop.hbase.regionserver.DefaultStoreEngine`.
   Either option has the same effect.
 +
@@ -2232,7 +2230,7 @@ This is not necessary on new tables.
 [[ops.stripe.config]]
 == Configuring Stripe Compaction
 
-Each of the settings for stripe compaction should be configured at the table 
or column family, after disabling the table.
+Each of the settings for stripe compaction should be configured at the table 
or column family level.
 If you use HBase shell, the general command pattern is as follows:
 
 [source,sql]

[14/50] hbase git commit: HBASE-15727 Canary Tool for Zookeeper (churro morales)

[syuanjiang]

HBASE-15727 Canary Tool for Zookeeper (churro morales)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7e5d5308
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7e5d5308
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7e5d5308

Branch: refs/heads/hbase-12439
Commit: 7e5d530870f146dfdee52e5a228ad84f0aefafd7
Parents: cd25880
Author: tedyu 
Authored: Thu Jun 2 10:15:08 2016 -0700
Committer: tedyu 
Committed: Thu Jun 2 10:15:08 2016 -0700

--
 .../org/apache/hadoop/hbase/tool/Canary.java| 142 ++-
 .../hadoop/hbase/tool/TestCanaryTool.java   |  24 +++-
 2 files changed, 160 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7e5d5308/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index ab9971d..360b0f5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -19,8 +19,14 @@
 
 package org.apache.hadoop.hbase.tool;
 
+import static 
org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
+
+import com.google.common.collect.Lists;
+
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -32,12 +38,12 @@ import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -78,20 +84,29 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.hadoop.hbase.util.RegionSplitter;
+import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.client.ConnectStringParser;
+import org.apache.zookeeper.data.Stat;
 
 /**
  * HBase Canary Tool, that that can be used to do
  * "canary monitoring" of a running HBase cluster.
  *
- * Here are two modes
+ * Here are three modes
  * 1. region mode - Foreach region tries to get one row per column family
  * and outputs some information about failure or latency.
  *
  * 2. regionserver mode - Foreach regionserver tries to get one row from one 
table
  * selected randomly and outputs some information about failure or latency.
+ *
+ * 3. zookeeper mode - for each zookeeper instance, selects a zNode and
+ * outputs some information about failure or latency.
  */
 public final class Canary implements Tool {
   // Sink interface used by the canary to outputs information
@@ -188,6 +203,55 @@ public final class Canary implements Tool {
 }
   }
 
+  public static class ZookeeperStdOutSink extends StdOutSink implements 
ExtendedSink {
+@Override public void publishReadFailure(String zNode, String server) {
+  incReadFailureCount();
+  LOG.error(String.format("Read from zNode:%s on zookeeper instance:%s", 
zNode, server));
+}
+
+@Override public void publishReadTiming(String znode, String server, long 
msTime) {
+  LOG.info(String.format("Read from zNode:%s on zookeeper instance:%s in 
%dms",
+  znode, server, msTime));
+}
+  }
+
+  static class ZookeeperTask implements Callable {
+private final Connection connection;
+private final String host;
+private String znode;
+private final int timeout;
+private ZookeeperStdOutSink sink;
+
+public ZookeeperTask(Connection connection, String host, String znode, int 
timeout,
+ZookeeperStdOutSink sink) {
+  this.connection = connection;
+  this.host = host;
+  this.znode = znode;
+  this.timeout = timeout;
+  this.sink = sink;
+}
+
+@Override public Void call() throws Exception {
+  ZooKeeper zooKeeper = null;
+  try {
+zooKeeper = new ZooKeeper(host, 

[46/50] hbase git commit: HBASE-15990 The priority value of subsequent coprocessors in the Coprocessor.Priority.SYSTEM list are not incremented by one (ChiaPing Tsai)

[syuanjiang]

HBASE-15990 The priority value of subsequent coprocessors in the 
Coprocessor.Priority.SYSTEM list are not incremented by one (ChiaPing Tsai)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55a04b78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55a04b78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55a04b78

Branch: refs/heads/hbase-12439
Commit: 55a04b78102a3a919c6e2e86fcdf98dd1d9a24e4
Parents: 9012a0b
Author: tedyu 
Authored: Thu Jun 9 10:26:10 2016 -0700
Committer: Ted 
Committed: Thu Jun 9 10:26:10 2016 -0700

--
 .../hbase/coprocessor/CoprocessorHost.java  |  5 ++--
 .../hbase/coprocessor/TestCoprocessorHost.java  | 26 ++--
 2 files changed, 21 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/55a04b78/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index da0e8b1..e937569 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -158,9 +158,10 @@ public abstract class CoprocessorHost {
 implClass = cl.loadClass(className);
 // Add coprocessors as we go to guard against case where a coprocessor 
is specified twice
 // in the configuration
-this.coprocessors.add(loadInstance(implClass, 
Coprocessor.PRIORITY_SYSTEM, conf));
+this.coprocessors.add(loadInstance(implClass, priority, conf));
 LOG.info("System coprocessor " + className + " was loaded " +
-"successfully with priority (" + priority++ + ").");
+"successfully with priority (" + priority + ").");
+++priority;
   } catch (Throwable t) {
 // We always abort if system coprocessors cannot be loaded
 abortServer(className, t);

http://git-wip-us.apache.org/repos/asf/hbase/blob/55a04b78/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
index 58cd0fb..66b5c60 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -51,9 +53,8 @@ public class TestCoprocessorHost {
   return this.aborted;
 }
   }
-
   @Test
-  public void testDoubleLoading() {
+  public void testDoubleLoadingAndPriorityValue() {
 final Configuration conf = HBaseConfiguration.create();
 CoprocessorHost host =
 new CoprocessorHost(new TestAbortable()) {
@@ -61,7 +62,7 @@ public class TestCoprocessorHost {
 
   @Override
   public CoprocessorEnvironment createEnvironment(Class implClass,
-  final Coprocessor instance, int priority, int sequence, 
Configuration conf) {
+  final Coprocessor instance, final int priority, int sequence, 
Configuration conf) {
 return new CoprocessorEnvironment() {
   final Coprocessor envInstance = instance;
 
@@ -82,7 +83,7 @@ public class TestCoprocessorHost {
 
   @Override
   public int getPriority() {
-return 0;
+return priority;
   }
 
   @Override
@@ -114,10 +115,19 @@ public class TestCoprocessorHost {
 };
 final String key = "KEY";
 final String coprocessor = 
"org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver";
-// Try and load coprocessor three times.
-conf.setStrings(key, coprocessor, coprocessor, coprocessor);
+// Try and load a coprocessor three times
+conf.setStrings(key, coprocessor, coprocessor, coprocessor, 
SimpleRegionObserverV2.class.getName());
 host.loadSystemCoprocessors(conf, key);
-// Only one coprocessor loaded
-Assert.assertEquals(1, host.coprocessors.size());
+// Two coprocessors(SimpleRegionObserver and SimpleRegionObserverV2) 

[42/50] hbase git commit: Revert "HBASE-15994 Allow selection of RpcSchedulers Adds logging by the RpcExecutors of their run configs Changes the default RpcSchedulerFactory from SimpleRpcSchedulerFact

[syuanjiang]

Revert "HBASE-15994 Allow selection of RpcSchedulers Adds logging by the 
RpcExecutors of their run configs Changes the default RpcSchedulerFactory from 
SimpleRpcSchedulerFactory.class to RpcSchedulerFactoryImpl.class. 
RpcSchedulerFactoryImpl.class. defaults to using 
SimpleRpcSchedulerFactory.class and the SimpleRpcScheduler, as has been default 
up to this, unless you set "hbase.region.server.rpc.scheduler.class" to 
org.apache.hadoop.hbase.ipc.FifoRpcScheduler"

This reverts commit 3ac4a57fd205e1909c874cabd7fda9fb176f3f0f.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/407aa4d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/407aa4d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/407aa4d4

Branch: refs/heads/hbase-12439
Commit: 407aa4d4963eb0d4aa3e0950babc8cec31367e11
Parents: 3ac4a57
Author: stack 
Authored: Wed Jun 8 20:22:50 2016 -0700
Committer: stack 
Committed: Wed Jun 8 20:22:50 2016 -0700

--
 .../hbase/ipc/BalancedQueueRpcExecutor.java |  4 --
 .../hadoop/hbase/ipc/FifoRpcScheduler.java  |  5 --
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java| 54 +++
 .../regionserver/FifoRpcSchedulerFactory.java   | 47 -
 .../hbase/regionserver/RpcSchedulerFactory.java |  4 +-
 .../regionserver/SimpleRpcSchedulerFactory.java |  6 +-
 .../regionserver/TestRpcSchedulerFactory.java   | 71 
 7 files changed, 30 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/407aa4d4/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
index 3505221..e4205eb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BalancedQueueRpcExecutor.java
@@ -22,8 +22,6 @@ import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
@@ -38,7 +36,6 @@ import org.apache.hadoop.hbase.util.ReflectionUtils;
 @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, 
HBaseInterfaceAudience.PHOENIX })
 @InterfaceStability.Evolving
 public class BalancedQueueRpcExecutor extends RpcExecutor {
-  private static final Log LOG = 
LogFactory.getLog(BalancedQueueRpcExecutor.class);
 
   protected final List queues;
   private final QueueBalancer balancer;
@@ -65,7 +62,6 @@ public class BalancedQueueRpcExecutor extends RpcExecutor {
 queues = new ArrayList(numQueues);
 this.balancer = getBalancer(numQueues);
 initializeQueues(numQueues, queueClass, initargs);
-LOG.debug(name + " queues=" + numQueues + " handlerCount=" + handlerCount);
   }
 
   protected void initializeQueues(final int numQueues,

http://git-wip-us.apache.org/repos/asf/hbase/blob/407aa4d4/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
index 70d903a..ee36f3f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FifoRpcScheduler.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.DaemonThreadFactory;
 
@@ -34,7 +32,6 @@ import java.util.concurrent.atomic.AtomicInteger;
  * This can be used for HMaster, where no prioritization is needed.
  */
 public class FifoRpcScheduler extends RpcScheduler {
-  private static final Log LOG = LogFactory.getLog(FifoRpcScheduler.class);
   private final int handlerCount;
   private final int maxQueueLength;
   private final AtomicInteger queueSize = new AtomicInteger(0);
@@ -44,8 +41,6 @@ public class FifoRpcScheduler extends RpcScheduler {
 this.handlerCount = handlerCount;
 this.maxQueueLength = 
conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH,
 handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
- 

[19/50] hbase git commit: HBASE-15845 Changes: - Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files right now which is confusing. - Remove omnipresence of formatter object si

[syuanjiang]

HBASE-15845 Changes:
- Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files 
right now which is confusing.
- Remove omnipresence of formatter object since it is kind of a use-and-throw 
class. Commands should create
  an instance, use it to format the output and discard it.
- Some refactoring

Change-Id: If9ea9873904e0a39d199a6aa10e23864b86a2f09


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdb46f01
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdb46f01
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdb46f01

Branch: refs/heads/hbase-12439
Commit: bdb46f01b9645a0aa4e9827a7e627de76cd7983b
Parents: f0c159b
Author: Apekshit 
Authored: Mon Dec 28 14:50:50 2015 -0800
Committer: Apekshit Sharma 
Committed: Fri Jun 3 13:37:38 2016 -0700

--
 bin/hirb.rb |   8 +-
 hbase-shell/src/main/ruby/hbase.rb  | 109 ---
 hbase-shell/src/main/ruby/hbase/admin.rb|  82 ++
 hbase-shell/src/main/ruby/hbase/hbase.rb|  36 +++---
 hbase-shell/src/main/ruby/hbase/quotas.rb   |   3 +-
 .../src/main/ruby/hbase/replication_admin.rb|   3 +-
 .../src/main/ruby/hbase/rsgroup_admin.rb|   3 +-
 hbase-shell/src/main/ruby/hbase/security.rb |   3 +-
 hbase-shell/src/main/ruby/hbase/table.rb|   2 +-
 hbase-shell/src/main/ruby/hbase/taskmonitor.rb  |   3 +-
 .../src/main/ruby/hbase/visibility_labels.rb|   5 +-
 hbase-shell/src/main/ruby/hbase_constants.rb| 109 +++
 hbase-shell/src/main/ruby/shell.rb  |  25 ++---
 hbase-shell/src/main/ruby/shell/commands.rb |   9 +-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   |   3 +-
 hbase-shell/src/test/ruby/hbase/hbase_test.rb   |  11 +-
 .../test/ruby/hbase/replication_admin_test.rb   |   3 +-
 .../src/test/ruby/hbase/security_admin_test.rb  |   3 +-
 hbase-shell/src/test/ruby/hbase/table_test.rb   |   2 +-
 .../src/test/ruby/hbase/taskmonitor_test.rb |   2 +-
 .../ruby/hbase/visibility_labels_admin_test.rb  |   3 +-
 .../src/test/ruby/shell/commands_test.rb|   2 +-
 .../src/test/ruby/shell/noninteractive_test.rb  |   6 +-
 .../src/test/ruby/shell/rsgroup_shell_test.rb   |  18 ++-
 hbase-shell/src/test/ruby/shell/shell_test.rb   |   6 +-
 hbase-shell/src/test/ruby/test_helper.rb|  10 +-
 26 files changed, 218 insertions(+), 251 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/bin/hirb.rb
--
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 94b5cdb..d0295d6 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -115,7 +115,7 @@ 
org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level)
 
org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
 
 # Require HBase now after setting log levels
-require 'hbase'
+require 'hbase_constants'
 
 # Load hbase shell
 require 'shell'
@@ -123,15 +123,11 @@ require 'shell'
 # Require formatter
 require 'shell/formatter'
 
-# Presume console format.
-# Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new
-
 # Setup the HBase module.  Create a configuration.
 @hbase = Hbase::Hbase.new
 
 # Setup console
-@shell = Shell::Shell.new(@hbase, @formatter, interactive)
+@shell = Shell::Shell.new(@hbase, interactive)
 @shell.debug = @shell_debug
 
 # Add commands to this namespace

http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/hbase-shell/src/main/ruby/hbase.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase.rb 
b/hbase-shell/src/main/ruby/hbase.rb
deleted file mode 100644
index bc6f37c..000
--- a/hbase-shell/src/main/ruby/hbase.rb
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# HBase ruby classes.
-# Has wrapper classes for org.apache.hadoop.hbase.client.Admin
-# 

[20/50] hbase git commit: HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragme

[syuanjiang]

HBASE-15949 Cleanup TestRegionServerMetrics.
@Before and @After to setup/teardown tables using @Rule to set table name based 
on testname.
Refactor out copy-pasted code fragments to single function.
(Apekshit)

Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b557f0be
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b557f0be
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b557f0be

Branch: refs/heads/hbase-12439
Commit: b557f0bec62a48753e5d01d7a47f3c9e5a6b3ee8
Parents: bdb46f0
Author: Apekshit 
Authored: Wed Jun 1 21:52:29 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 14:27:24 2016 -0700

--
 .../regionserver/TestRegionServerMetrics.java   | 682 +++
 1 file changed, 256 insertions(+), 426 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b557f0be/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 18796bd..06db468 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.*;
@@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
 
 import static org.junit.Assert.*;
 
@@ -39,23 +47,35 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-
 @Category({RegionServerTests.class, MediumTests.class})
 public class TestRegionServerMetrics {
-  private static MetricsAssertHelper metricsHelper;
+  private static final Log LOG = 
LogFactory.getLog(TestRegionServerMetrics.class);
+
+  @Rule
+  public TestName testName = new TestName();
+
+  @ClassRule
+  public static TestRule timeout = 
CategoryBasedTimeout.forClass(TestRegionServerMetrics.class);
 
   static {
 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
   }
 
+  private static MetricsAssertHelper metricsHelper;
   private static MiniHBaseCluster cluster;
   private static HRegionServer rs;
   private static Configuration conf;
   private static HBaseTestingUtility TEST_UTIL;
+  private static Connection connection;
   private static MetricsRegionServer metricsRegionServer;
   private static MetricsRegionServerSource serverSource;
   private static final int NUM_SCAN_NEXT = 30;
   private static int numScanNext = 0;
+  private static byte[] cf = Bytes.toBytes("cf");
+  private static byte[] row = Bytes.toBytes("row");
+  private static byte[] qualifier = Bytes.toBytes("qual");
+  private static byte[] val = Bytes.toBytes("val");
+  private static Admin admin;
 
   @BeforeClass
   public static void startCluster() throws Exception {
@@ -65,12 +85,16 @@ public class TestRegionServerMetrics {
 conf.getLong("hbase.splitlog.max.resubmit", 0);
 // Make the failure test faster
 conf.setInt("zookeeper.recovery.retry", 0);
+// testMobMetrics creates few hfiles and manages compaction manually.
+conf.setInt("hbase.hstore.compactionThreshold", 100);
+conf.setInt("hbase.hstore.compaction.max", 100);
 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
 
 TEST_UTIL.startMiniCluster(1, 1);
 cluster = TEST_UTIL.getHBaseCluster();
-
 cluster.waitForActiveAndReadyMaster();
+admin = TEST_UTIL.getHBaseAdmin();
+connection = TEST_UTIL.getConnection();
 
 while (cluster.getLiveRegionServerThreads().size() < 1) {
   Threads.sleep(100);
@@ -88,551 +112,370 @@ public class TestRegionServerMetrics {
 }
   }
 
-  @Test(timeout = 30)
+  TableName tableName;
+  Table table;
+
+  @Before
+  public void beforeTestMethod() throws Exception {
+metricsRegionServer.getRegionServerWrapper().forceRecompute();
+tableName = 

[39/50] hbase git commit: HBASE-15989 Remove hbase.online.schema.update.enable

[syuanjiang]

HBASE-15989 Remove hbase.online.schema.update.enable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d9463bcc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d9463bcc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d9463bcc

Branch: refs/heads/hbase-12439
Commit: d9463bcce0e36bfb67c82acd1d7483f63b2764b7
Parents: d5d9b7d
Author: Matteo Bertozzi 
Authored: Wed Jun 8 13:09:31 2016 -0700
Committer: Matteo Bertozzi 
Committed: Wed Jun 8 13:09:31 2016 -0700

--
 .../client/replication/ReplicationAdmin.java| 13 +-
 .../src/main/resources/hbase-default.xml|  9 +
 .../procedure/MasterDDLOperationHelper.java | 15 ---
 .../master/procedure/ModifyTableProcedure.java  |  5 ---
 .../apache/hadoop/hbase/client/TestAdmin1.java  | 42 
 .../apache/hadoop/hbase/client/TestAdmin2.java  |  1 -
 .../client/TestCloneSnapshotFromClient.java |  1 -
 .../hbase/client/TestFromClientSide3.java   |  2 -
 .../client/TestRestoreSnapshotFromClient.java   |  1 -
 .../hbase/io/encoding/TestChangingEncoding.java |  1 -
 .../hbase/master/TestTableLockManager.java  |  1 -
 .../regionserver/TestEncryptionKeyRotation.java |  4 +-
 ...sibilityLabelReplicationWithExpAsString.java |  1 -
 .../TestVisibilityLabelsReplication.java|  1 -
 ...ibilityLabelsWithDefaultVisLabelService.java |  1 -
 .../TestRestoreFlushSnapshotFromClient.java |  1 -
 .../src/main/ruby/shell/commands/alter.rb   | 16 
 .../hadoop/hbase/client/AbstractTestShell.java  |  1 -
 .../hbase/client/rsgroup/TestShellRSGroups.java |  1 -
 src/main/asciidoc/_chapters/hbase-default.adoc  | 10 -
 20 files changed, 12 insertions(+), 115 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d9463bcc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index a2ad2e7..d062448 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -516,9 +516,9 @@ public class ReplicationAdmin implements Closeable {
 if (repPeers == null || repPeers.size() <= 0) {
   throw new IllegalArgumentException("Found no peer cluster for 
replication.");
 }
-
+
 final TableName onlyTableNameQualifier = 
TableName.valueOf(tableName.getQualifierAsString());
-
+
 for (ReplicationPeer repPeer : repPeers) {
   Map tableCFMap = repPeer.getTableCFs();
   // TODO Currently peer TableCFs will not include namespace so we need to 
check only for table
@@ -595,20 +595,11 @@ public class ReplicationAdmin implements Closeable {
   admin = this.connection.getAdmin();
   HTableDescriptor htd = admin.getTableDescriptor(tableName);
   if (isTableRepEnabled(htd) ^ isRepEnabled) {
-boolean isOnlineSchemaUpdateEnabled =
-this.connection.getConfiguration()
-.getBoolean("hbase.online.schema.update.enable", true);
-if (!isOnlineSchemaUpdateEnabled) {
-  admin.disableTable(tableName);
-}
 for (HColumnDescriptor hcd : htd.getFamilies()) {
   hcd.setScope(isRepEnabled ? HConstants.REPLICATION_SCOPE_GLOBAL
   : HConstants.REPLICATION_SCOPE_LOCAL);
 }
 admin.modifyTable(tableName, htd);
-if (!isOnlineSchemaUpdateEnabled) {
-  admin.enableTable(tableName);
-}
   }
 } finally {
   if (admin != null) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/d9463bcc/hbase-common/src/main/resources/hbase-default.xml
--
diff --git a/hbase-common/src/main/resources/hbase-default.xml 
b/hbase-common/src/main/resources/hbase-default.xml
index 62a6b62..55ac497 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -562,7 +562,7 @@ possible configurations would overwhelm and obscure the 
important.
   
 hbase.regions.slop
 0.001
-Rebalance if any regionserver has average + (average * slop) 
regions. 
+Rebalance if any regionserver has average + (average * slop) 
regions.
   The default value of this parameter is 0.001 in StochasticLoadBalancer 
(the default load balancer),
   while the default is 0.2 in other load balancers (i.e., 
SimpleLoadBalancer).
   
@@ -865,7 

[34/50] hbase git commit: Revert "HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count" Revert mistaken commit This reverts commit 1125215aad3f5b149f3458ba7019c5920f6dca66.

[syuanjiang]

Revert "HBASE-15967 Metric for active ipc Readers and make default fraction of 
cpu count"
Revert mistaken commit
This reverts commit 1125215aad3f5b149f3458ba7019c5920f6dca66.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d5a2593
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d5a2593
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d5a2593

Branch: refs/heads/hbase-12439
Commit: 6d5a25935e5ce983e14eff576a699ed1948566d2
Parents: 2da090f
Author: stack 
Authored: Tue Jun 7 16:41:01 2016 -0700
Committer: stack 
Committed: Tue Jun 7 16:41:01 2016 -0700

--
 .../hbase/ipc/MetricsHBaseServerSource.java | 11 +++---
 .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 19 
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  | 23 +---
 3 files changed, 8 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
index 43515cd..ce57e0f 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
@@ -86,13 +86,6 @@ public interface MetricsHBaseServerSource extends BaseSource 
{
   String EXCEPTIONS_MULTI_TOO_LARGE_DESC = "A response to a multi request was 
too large and the " +
   "rest of the requests will have to be retried.";
 
-  String RUNNING_READERS = "runningReaders";
-  String RUNNING_READERS_DESCRIPTION =
-  "Count of Reader threads currently busy parsing requests to hand off to 
the scheduler";
-
-  void incrRunningReaders();
-  void decrRunningReaders();
-
   void authorizationSuccess();
 
   void authorizationFailure();
@@ -129,4 +122,6 @@ public interface MetricsHBaseServerSource extends 
BaseSource {
   void processedCall(int processingTime);
 
   void queuedAndProcessedCall(int totalTime);
-}
\ No newline at end of file
+
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
--
diff --git 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
index 24cc0fb..c72641d 100644
--- 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
+++ 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
@@ -57,12 +57,6 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
   private MetricHistogram requestSize;
   private MetricHistogram responseSize;
 
-  /**
-   * The count of readers currently working parsing a request as opposed to 
being blocked on the
-   * selector waiting on requests to come in.
-   */
-  private final MutableFastCounter runningReaders;
-
   public MetricsHBaseServerSourceImpl(String metricsName,
   String metricsDescription,
   String metricsContext,
@@ -92,9 +86,6 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
 this.exceptionsMultiTooLarge = this.getMetricsRegistry()
 .newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, 
EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L);
 
-this.runningReaders = this.getMetricsRegistry()
-.newCounter(RUNNING_READERS, RUNNING_READERS_DESCRIPTION, 0L);
-
 this.authenticationSuccesses = this.getMetricsRegistry().newCounter(
 AUTHENTICATION_SUCCESSES_NAME, AUTHENTICATION_SUCCESSES_DESC, 0L);
 this.authenticationFailures = 
this.getMetricsRegistry().newCounter(AUTHENTICATION_FAILURES_NAME,
@@ -118,16 +109,6 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
   }
 
   @Override
-  public void incrRunningReaders() {
-this.runningReaders.incr(+1);
-  }
-
-  @Override
-  public void decrRunningReaders() {
-this.runningReaders.incr(-1);
-  }
-
-  @Override
   public void authorizationSuccess() {
 authorizationSuccesses.incr();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/6d5a2593/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 

[15/50] hbase git commit: HBASE-15881 Allow BZIP2 compression.

[syuanjiang]

HBASE-15881 Allow BZIP2 compression.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc890a2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc890a2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc890a2e

Branch: refs/heads/hbase-12439
Commit: fc890a2ecb12c3c664ad00f8b2f8788dba04c71d
Parents: 7e5d530
Author: Lars Hofhansl 
Authored: Thu Jun 2 12:19:02 2016 -0700
Committer: Lars Hofhansl 
Committed: Thu Jun 2 12:19:02 2016 -0700

--
 .../hadoop/hbase/io/compress/Compression.java   | 29 +++-
 .../hadoop/hbase/util/TestCompressionTest.java  |  2 ++
 2 files changed, 30 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fc890a2e/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index 821b21f..6dc4190 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
   throw new RuntimeException(e);
 }
   }
-  };
+},
+BZIP2("bzip2") {
+  // Use base type to avoid compile-time dependencies.
+  private volatile transient CompressionCodec bzipCodec;
+  private transient Object lock = new Object();
+
+  @Override
+  CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+  synchronized (lock) {
+if (bzipCodec == null) {
+  bzipCodec = buildCodec(conf);
+}
+  }
+}
+return bzipCodec;
+  }
+
+  private CompressionCodec buildCodec(Configuration conf) {
+try {
+  Class externalCodec =
+  
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+  return (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
conf);
+} catch (ClassNotFoundException e) {
+  throw new RuntimeException(e);
+}
+  }
+};
 
 private final Configuration conf;
 private final String compressName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/fc890a2e/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index 43c7cfc..398f3f0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -75,12 +75,14 @@ public class TestCompressionTest {
   nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
   nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
   nativeCodecTest("SNAPPY", "snappy", 
"org.apache.hadoop.io.compress.SnappyCodec");
+  nativeCodecTest("BZIP2", "bzip2", 
"org.apache.hadoop.io.compress.BZip2Codec");
 } else {
   // Hadoop nativelib is not available
   LOG.debug("Native code not loaded");
   assertFalse(CompressionTest.testCompression("LZO"));
   assertFalse(CompressionTest.testCompression("LZ4"));
   assertFalse(CompressionTest.testCompression("SNAPPY"));
+  assertFalse(CompressionTest.testCompression("BZIP2"));
 }
   }
 

[23/50] hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

[syuanjiang]

HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de1b5ff7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de1b5ff7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de1b5ff7

Branch: refs/heads/hbase-12439
Commit: de1b5ff7761b899691085e147c885e15897a562c
Parents: 21e9827
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:49:23 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/de1b5ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index b9abc65..c93b307 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -86,7 +87,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

[32/50] hbase git commit: HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count Add new metric hbase.regionserver.ipc.runningReaders Also make it so Reader count is a factor

[syuanjiang]

HBASE-15967 Metric for active ipc Readers and make default fraction of cpu count
Add new metric hbase.regionserver.ipc.runningReaders
Also make it so Reader count is a factor of processor count


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1125215a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1125215a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1125215a

Branch: refs/heads/hbase-12439
Commit: 1125215aad3f5b149f3458ba7019c5920f6dca66
Parents: e0b70c0
Author: stack 
Authored: Sun Jun 5 11:12:05 2016 -0700
Committer: stack 
Committed: Tue Jun 7 13:10:14 2016 -0700

--
 .../hbase/ipc/MetricsHBaseServerSource.java | 11 +++---
 .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 19 
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  | 23 +++-
 3 files changed, 45 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
index ce57e0f..43515cd 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
@@ -86,6 +86,13 @@ public interface MetricsHBaseServerSource extends BaseSource 
{
   String EXCEPTIONS_MULTI_TOO_LARGE_DESC = "A response to a multi request was 
too large and the " +
   "rest of the requests will have to be retried.";
 
+  String RUNNING_READERS = "runningReaders";
+  String RUNNING_READERS_DESCRIPTION =
+  "Count of Reader threads currently busy parsing requests to hand off to 
the scheduler";
+
+  void incrRunningReaders();
+  void decrRunningReaders();
+
   void authorizationSuccess();
 
   void authorizationFailure();
@@ -122,6 +129,4 @@ public interface MetricsHBaseServerSource extends 
BaseSource {
   void processedCall(int processingTime);
 
   void queuedAndProcessedCall(int totalTime);
-
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
--
diff --git 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
index c72641d..24cc0fb 100644
--- 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
+++ 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
@@ -57,6 +57,12 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
   private MetricHistogram requestSize;
   private MetricHistogram responseSize;
 
+  /**
+   * The count of readers currently working parsing a request as opposed to 
being blocked on the
+   * selector waiting on requests to come in.
+   */
+  private final MutableFastCounter runningReaders;
+
   public MetricsHBaseServerSourceImpl(String metricsName,
   String metricsDescription,
   String metricsContext,
@@ -86,6 +92,9 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
 this.exceptionsMultiTooLarge = this.getMetricsRegistry()
 .newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, 
EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L);
 
+this.runningReaders = this.getMetricsRegistry()
+.newCounter(RUNNING_READERS, RUNNING_READERS_DESCRIPTION, 0L);
+
 this.authenticationSuccesses = this.getMetricsRegistry().newCounter(
 AUTHENTICATION_SUCCESSES_NAME, AUTHENTICATION_SUCCESSES_DESC, 0L);
 this.authenticationFailures = 
this.getMetricsRegistry().newCounter(AUTHENTICATION_FAILURES_NAME,
@@ -109,6 +118,16 @@ public class MetricsHBaseServerSourceImpl extends 
BaseSourceImpl
   }
 
   @Override
+  public void incrRunningReaders() {
+this.runningReaders.incr(+1);
+  }
+
+  @Override
+  public void decrRunningReaders() {
+this.runningReaders.incr(-1);
+  }
+
+  @Override
   public void authorizationSuccess() {
 authorizationSuccesses.incr();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/1125215a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 

[49/50] hbase git commit: HBASE-15946. Eliminate possible security concerns in Store File metrics.

[syuanjiang]

HBASE-15946. Eliminate possible security concerns in Store File metrics.

Invoking 'hbase hfile' inside a servlet raises several concerns. This
patch avoids invoking a separate process, and also adds validation that
the file being read is at least inside the HBase root directory.

Signed-off-by: Mikhail Antonov 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6da6babe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6da6babe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6da6babe

Branch: refs/heads/hbase-12439
Commit: 6da6babe4faa7b2b16775d3cd5c861e71ef4cf31
Parents: babdedc
Author: Sean Mackrory 
Authored: Tue May 31 10:28:27 2016 -0600
Committer: Mikhail Antonov 
Committed: Thu Jun 9 16:08:19 2016 -0700

--
 .../hbase/io/hfile/HFilePrettyPrinter.java  | 108 ---
 .../hbase-webapps/regionserver/storeFile.jsp|  35 +++---
 2 files changed, 83 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6da6babe/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index e9e21fe..36067e5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -1,4 +1,3 @@
-
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -115,6 +114,8 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
   private Map mobFileLocations;
   private static final int FOUND_MOB_FILES_CACHE_CAPACITY = 50;
   private static final int MISSING_MOB_FILES_CACHE_CAPACITY = 20;
+  private PrintStream out = System.out;
+  private PrintStream err = System.err;
 
   /**
* The row which the user wants to specify and print all the KeyValues for.
@@ -161,6 +162,11 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 options.addOptionGroup(files);
   }
 
+  public void setPrintStreams(PrintStream out, PrintStream err) {
+this.out = out;
+this.err = err;
+  }
+
   public boolean parseOptions(String args[]) throws ParseException,
   IOException {
 if (args.length == 0) {
@@ -192,7 +198,7 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 row = Bytes.toBytesBinary(key);
 isSeekToRow = true;
   } else {
-System.err.println("Invalid row is specified.");
+err.println("Invalid row is specified.");
 System.exit(-1);
   }
 }
@@ -206,17 +212,17 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
   String enc = HRegionInfo.encodeRegionName(rn);
   Path regionDir = new Path(tableDir, enc);
   if (verbose)
-System.out.println("region dir -> " + regionDir);
+out.println("region dir -> " + regionDir);
   List regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()),
   regionDir);
   if (verbose)
-System.out.println("Number of region files found -> "
+out.println("Number of region files found -> "
 + regionFiles.size());
   if (verbose) {
 int i = 1;
 for (Path p : regionFiles) {
   if (verbose)
-System.out.println("Found file[" + i++ + "] -> " + p);
+out.println("Found file[" + i++ + "] -> " + p);
 }
   }
   files.addAll(regionFiles);
@@ -255,27 +261,46 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 // iterate over all files found
 for (Path fileName : files) {
   try {
-processFile(fileName);
+int exitCode = processFile(fileName);
+if (exitCode != 0) {
+  return exitCode;
+}
   } catch (IOException ex) {
 LOG.error("Error reading " + fileName, ex);
-System.exit(-2);
+return -2;
   }
 }
 
 if (verbose || printKey) {
-  System.out.println("Scanned kv count -> " + count);
+  out.println("Scanned kv count -> " + count);
 }
 
 return 0;
   }
 
-  private void processFile(Path file) throws IOException {
+  public int processFile(Path file) throws IOException {
 if (verbose)
-  System.out.println("Scanning -> " + file);
+  out.println("Scanning -> " + file);
+
+Path rootPath = FSUtils.getRootDir(getConf());
+String rootString = rootPath + rootPath.SEPARATOR;
+if (!file.toString().startsWith(rootString)) {
+  // First we see if 

[24/50] hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They ar

[syuanjiang]

HBASE-15849 Simplify the way we handle runtime of commands.
Functions format_simple_command and format_and_return_simple_command are used 
to print runtimes right now. They are called from within every single command 
and use Ruby's 'yield' magic.  Instead, we can simplify it using 'command_safe' 
function. Since command_safe wraps all commands, we can simply time before and 
after we call individual command.
If a command only wants to time a part of its logic, it can set instance 
variables start_time and end_time accordingly which is far more simpler to 
understand and work with than 'yield'.

Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70762faa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70762faa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70762faa

Branch: refs/heads/hbase-12439
Commit: 70762faa98e6ad26f10fa8bd9ebc1824fdcb507c
Parents: de1b5ff
Author: Apekshit Sharma 
Authored: Fri Jun 3 14:13:45 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 15:50:03 2016 -0700

--
 hbase-shell/src/main/ruby/shell.rb  |  7 ++---
 hbase-shell/src/main/ruby/shell/commands.rb | 27 +++-
 .../main/ruby/shell/commands/abort_procedure.rb |  6 +
 .../src/main/ruby/shell/commands/add_labels.rb  |  4 +--
 .../src/main/ruby/shell/commands/add_peer.rb|  4 +--
 .../src/main/ruby/shell/commands/alter.rb   |  4 +--
 .../src/main/ruby/shell/commands/alter_async.rb |  4 +--
 .../main/ruby/shell/commands/alter_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/append.rb  | 10 
 .../ruby/shell/commands/append_peer_tableCFs.rb |  4 +--
 .../src/main/ruby/shell/commands/assign.rb  |  4 +--
 .../main/ruby/shell/commands/balance_switch.rb  |  6 +
 .../src/main/ruby/shell/commands/balancer.rb| 14 +-
 .../ruby/shell/commands/balancer_enabled.rb |  6 +
 .../shell/commands/catalogjanitor_enabled.rb|  6 +
 .../ruby/shell/commands/catalogjanitor_run.rb   |  4 +--
 .../shell/commands/catalogjanitor_switch.rb |  6 +
 .../src/main/ruby/shell/commands/clear_auths.rb |  4 +--
 .../main/ruby/shell/commands/clone_snapshot.rb  |  4 +--
 .../main/ruby/shell/commands/close_region.rb|  4 +--
 .../src/main/ruby/shell/commands/compact.rb |  4 +--
 .../src/main/ruby/shell/commands/compact_rs.rb  |  4 +--
 .../src/main/ruby/shell/commands/count.rb   |  4 +--
 .../src/main/ruby/shell/commands/create.rb  |  7 +++--
 .../ruby/shell/commands/create_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/delete.rb  | 13 +-
 .../ruby/shell/commands/delete_all_snapshot.rb  |  6 ++---
 .../main/ruby/shell/commands/delete_snapshot.rb |  4 +--
 .../shell/commands/delete_table_snapshots.rb| 19 +++---
 .../src/main/ruby/shell/commands/deleteall.rb   |  5 ++--
 .../src/main/ruby/shell/commands/describe.rb|  4 +--
 .../ruby/shell/commands/describe_namespace.rb   |  3 ---
 .../src/main/ruby/shell/commands/disable.rb |  4 +--
 .../main/ruby/shell/commands/disable_peer.rb|  4 +--
 .../shell/commands/disable_table_replication.rb |  6 ++---
 .../src/main/ruby/shell/commands/drop.rb|  4 +--
 .../main/ruby/shell/commands/drop_namespace.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable_peer.rb |  4 +--
 .../shell/commands/enable_table_replication.rb  |  6 ++---
 .../src/main/ruby/shell/commands/exists.rb  |  4 +--
 .../src/main/ruby/shell/commands/flush.rb   |  4 +--
 hbase-shell/src/main/ruby/shell/commands/get.rb |  4 +--
 .../src/main/ruby/shell/commands/get_auths.rb   |  8 +++---
 .../main/ruby/shell/commands/get_peer_config.rb |  7 +++--
 .../src/main/ruby/shell/commands/get_table.rb   |  4 +--
 .../src/main/ruby/shell/commands/grant.rb   |  9 +++
 .../src/main/ruby/shell/commands/incr.rb| 12 -
 .../src/main/ruby/shell/commands/is_disabled.rb |  8 ++
 .../src/main/ruby/shell/commands/is_enabled.rb  |  6 +
 .../src/main/ruby/shell/commands/list.rb|  3 +--
 .../src/main/ruby/shell/commands/list_labels.rb |  8 +++---
 .../main/ruby/shell/commands/list_namespace.rb  |  3 +--
 .../shell/commands/list_namespace_tables.rb |  3 +--
 .../ruby/shell/commands/list_peer_configs.rb| 18 ++---
 .../src/main/ruby/shell/commands/list_peers.rb  |  3 +--
 .../main/ruby/shell/commands/list_procedures.rb |  3 +--
 .../src/main/ruby/shell/commands/list_quotas.rb |  3 +--
 .../shell/commands/list_replicated_tables.rb|  4 +--
 .../main/ruby/shell/commands/list_snapshots.rb  |  3 +--
 .../ruby/shell/commands/list_table_snapshots.rb |  3 +--
 .../main/ruby/shell/commands/locate_region.rb   |  4 +--
 .../main/ruby/shell/commands/major_compact.rb  

[45/50] hbase git commit: HBASE-15952 Bulk load data replication is not working when RS user does not have permission on hfile-refs node

[syuanjiang]

HBASE-15952 Bulk load data replication is not working when RS user does not 
have permission on hfile-refs node


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9012a0b1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9012a0b1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9012a0b1

Branch: refs/heads/hbase-12439
Commit: 9012a0b123b3eea8b08c8687cef812e83e9b491d
Parents: 41cc215
Author: Ashish Singhi 
Authored: Thu Jun 9 18:44:29 2016 +0530
Committer: Ashish Singhi 
Committed: Thu Jun 9 18:44:29 2016 +0530

--
 .../replication/ReplicationPeersZKImpl.java | 21 -
 .../hbase/replication/ReplicationQueues.java|  6 
 .../replication/ReplicationQueuesHBaseImpl.java |  6 
 .../replication/ReplicationQueuesZKImpl.java| 33 
 .../regionserver/ReplicationSourceManager.java  | 11 +--
 .../cleaner/TestReplicationHFileCleaner.java|  1 +
 .../replication/TestReplicationStateBasic.java  |  5 +++
 7 files changed, 53 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 15265d9..5af97c2 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -129,17 +129,6 @@ public class ReplicationPeersZKImpl extends 
ReplicationStateZKBase implements Re
 
   ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
 
-  // Irrespective of bulk load hfile replication is enabled or not we add 
peerId node to
-  // hfile-refs node -- HBASE-15397
-  try {
-String peerId = ZKUtil.joinZNode(this.hfileRefsZNode, id);
-LOG.info("Adding peer " + peerId + " to hfile reference queue.");
-ZKUtil.createWithParents(this.zookeeper, peerId);
-  } catch (KeeperException e) {
-throw new ReplicationException("Failed to add peer with id=" + id
-+ ", node under hfile references node.", e);
-  }
-
   List listOfOps = new ArrayList();
   ZKUtilOp op1 = ZKUtilOp.createAndFailSilent(getPeerNode(id),
 ReplicationSerDeHelper.toByteArray(peerConfig));
@@ -166,16 +155,6 @@ public class ReplicationPeersZKImpl extends 
ReplicationStateZKBase implements Re
 + " because that id does not exist.");
   }
   ZKUtil.deleteNodeRecursively(this.zookeeper, 
ZKUtil.joinZNode(this.peersZNode, id));
-  // Delete peerId node from hfile-refs node irrespective of whether bulk 
loaded hfile
-  // replication is enabled or not
-
-  String peerId = ZKUtil.joinZNode(this.hfileRefsZNode, id);
-  try {
-LOG.info("Removing peer " + peerId + " from hfile reference queue.");
-ZKUtil.deleteNodeRecursively(this.zookeeper, peerId);
-  } catch (NoNodeException e) {
-LOG.info("Did not find node " + peerId + " to delete.", e);
-  }
 } catch (KeeperException e) {
   throw new ReplicationException("Could not remove peer with id=" + id, e);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index db6da91..809b122 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -123,6 +123,12 @@ public interface ReplicationQueues {
   void addPeerToHFileRefs(String peerId) throws ReplicationException;
 
   /**
+   * Remove a peer from hfile reference queue.
+   * @param peerId peer cluster id to be removed
+   */
+  void removePeerFromHFileRefs(String peerId);
+
+  /**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
* @param files list of hfile references to be added

http://git-wip-us.apache.org/repos/asf/hbase/blob/9012a0b1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
--
diff --git 

[37/50] hbase git commit: HBASE-15600 Add provision for adding mutations to memstore or able to write to same region in batchMutate coprocessor hooks (Rajeshbabu and Enis)

[syuanjiang]

HBASE-15600 Add provision for adding mutations to memstore or able to write to 
same region in batchMutate coprocessor hooks (Rajeshbabu and Enis)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d05a3722
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d05a3722
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d05a3722

Branch: refs/heads/hbase-12439
Commit: d05a3722c8347363eb04a3e5457d13ae5d0d6de6
Parents: 3a95552
Author: Enis Soztutar 
Authored: Tue Jun 7 20:12:51 2016 -0700
Committer: Enis Soztutar 
Committed: Tue Jun 7 20:12:51 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  |  76 -
 .../MiniBatchOperationInProgress.java   |  29 +-
 .../regionserver/MultiRowMutationProcessor.java |   2 +-
 ...erverForAddingMutationsFromCoprocessors.java | 282 +++
 4 files changed, 370 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d05a3722/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 6522fde..9c966cd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3134,6 +3134,35 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, 
firstIndex, lastIndexExclusive);
 if (coprocessorHost.preBatchMutate(miniBatchOp)) {
   return 0L;
+} else {
+  for (int i = firstIndex; i < lastIndexExclusive; i++) {
+if (batchOp.retCodeDetails[i].getOperationStatusCode() != 
OperationStatusCode.NOT_RUN) {
+  // lastIndexExclusive was incremented above.
+  continue;
+}
+// we pass (i - firstIndex) below since the call expects a 
relative index
+Mutation[] cpMutations = 
miniBatchOp.getOperationsFromCoprocessors(i - firstIndex);
+if (cpMutations == null) {
+  continue;
+}
+// Else Coprocessor added more Mutations corresponding to the 
Mutation at this index.
+for (int j = 0; j < cpMutations.length; j++) {
+  Mutation cpMutation = cpMutations[j];
+  Map cpFamilyMap = 
cpMutation.getFamilyCellMap();
+  checkAndPrepareMutation(cpMutation, replay, cpFamilyMap, now);
+
+  // Acquire row locks. If not, the whole batch will fail.
+  acquiredRowLocks.add(getRowLockInternal(cpMutation.getRow(), 
true));
+
+  if (cpMutation.getDurability() == Durability.SKIP_WAL) {
+recordMutationWithoutWal(cpFamilyMap);
+  }
+
+  // Returned mutations from coprocessor correspond to the 
Mutation at index i. We can
+  // directly add the cells from those mutations to the familyMaps 
of this mutation.
+  mergeFamilyMaps(familyMaps[i], cpFamilyMap); // will get added 
to the memstore later
+}
+  }
 }
   }
 
@@ -3310,9 +3339,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 // call the coprocessor hook to do any finalization steps
 // after the put is done
 MiniBatchOperationInProgress miniBatchOp =
-new 
MiniBatchOperationInProgress(batchOp.getMutationsForCoprocs(),
-batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, 
firstIndex,
-lastIndexExclusive);
+  new 
MiniBatchOperationInProgress(batchOp.getMutationsForCoprocs(),
+  batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, 
firstIndex, lastIndexExclusive);
 coprocessorHost.postBatchMutateIndispensably(miniBatchOp, success);
   }
 
@@ -3320,6 +3348,18 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
   }
 
+  private void mergeFamilyMaps(Map familyMap,
+  Map toBeMerged) {
+for (Map.Entry entry : toBeMerged.entrySet()) {
+  List cells = familyMap.get(entry.getKey());
+  if (cells == null) {
+familyMap.put(entry.getKey(), entry.getValue());
+  } else {
+cells.addAll(entry.getValue());
+  }
+}
+  }
+
   private void appendCurrentNonces(final Mutation mutation, final boolean 
replay,
   final WALEdit walEdit, final long now, final long currentNonceGroup, 
final long currentNonce)
   

[28/50] hbase git commit: HBASE-15803 ZooKeeperWatcher's constructor can leak a ZooKeeper instance with throwing ZooKeeperConnectionException when canCreateBaseZNode is true

[syuanjiang]

HBASE-15803 ZooKeeperWatcher's constructor can leak a ZooKeeper instance with 
throwing ZooKeeperConnectionException when canCreateBaseZNode is true


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7fd3532d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7fd3532d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7fd3532d

Branch: refs/heads/hbase-12439
Commit: 7fd3532de63d7b1885d6993c11a35c2f85e26631
Parents: 15c03fd
Author: tedyu 
Authored: Mon Jun 6 18:35:15 2016 -0700
Committer: tedyu 
Committed: Mon Jun 6 18:35:15 2016 -0700

--
 .../apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7fd3532d/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 93828eb..ff3d1c7 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -171,7 +171,17 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, 
identifier);
 pendingWatcher.prepare(this);
 if (canCreateBaseZNode) {
-  createBaseZNodes();
+  try {
+createBaseZNodes();
+  } catch (ZooKeeperConnectionException zce) {
+try {
+  this.recoverableZooKeeper.close();
+} catch (InterruptedException ie) {
+  LOG.debug("Encountered InterruptedException when closing " + 
this.recoverableZooKeeper);
+  Thread.currentThread().interrupt();
+}
+throw zce;
+  }
 }
   }
 

[38/50] hbase git commit: HBASE-15107 Procedure v2 - Procedure Queue with Region locks

[syuanjiang]

HBASE-15107 Procedure v2 - Procedure Queue with Region locks


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d5d9b7d5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d5d9b7d5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d5d9b7d5

Branch: refs/heads/hbase-12439
Commit: d5d9b7d500c4e2bdf67abed462966f7bf7df
Parents: d05a372
Author: Matteo Bertozzi 
Authored: Wed Jun 8 12:52:58 2016 -0700
Committer: Matteo Bertozzi 
Committed: Wed Jun 8 12:52:58 2016 -0700

--
 .../hadoop/hbase/procedure2/Procedure.java  |  31 +-
 .../hbase/procedure2/ProcedureExecutor.java |  19 +-
 .../procedure2/ProcedureSuspendedException.java |  39 +++
 .../hbase/procedure2/SequentialProcedure.java   |   2 +-
 .../procedure/MasterProcedureScheduler.java | 286 +--
 .../procedure/TableProcedureInterface.java  |   1 +
 .../procedure/TestMasterProcedureScheduler.java | 228 ++-
 7 files changed, 580 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d5d9b7d5/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
--
diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 7e58420..ee61841 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -79,6 +79,9 @@ public abstract class Procedure implements 
Comparable {
   private int childrenLatch = 0;
   private long lastUpdate;
 
+  // TODO: it will be nice having pointers to allow the scheduler doing 
suspend/resume tricks
+  private boolean suspended = false;
+
   private RemoteProcedureException exception = null;
   private byte[] result = null;
 
@@ -94,7 +97,7 @@ public abstract class Procedure implements 
Comparable {
* @throws InterruptedException the procedure will be added back to the 
queue and retried later
*/
   protected abstract Procedure[] execute(TEnvironment env)
-throws ProcedureYieldException, InterruptedException;
+throws ProcedureYieldException, ProcedureSuspendedException, 
InterruptedException;
 
   /**
* The code to undo what done by the execute() code.
@@ -276,6 +279,9 @@ public abstract class Procedure implements 
Comparable {
*/
   protected void toStringState(StringBuilder builder) {
 builder.append(getState());
+if (isSuspended()) {
+  builder.append("|SUSPENDED");
+}
   }
 
   /**
@@ -319,7 +325,7 @@ public abstract class Procedure implements 
Comparable {
   }
 
   public long getParentProcId() {
-return parentProcId;
+return parentProcId.longValue();
   }
 
   public NonceKey getNonceKey() {
@@ -371,6 +377,23 @@ public abstract class Procedure implements 
Comparable {
 return false;
   }
 
+  /**
+   * @return true if the procedure is in a suspended state,
+   * waiting for the resources required to execute the procedure will 
become available.
+   */
+  public synchronized boolean isSuspended() {
+return suspended;
+  }
+
+  public synchronized void suspend() {
+suspended = true;
+  }
+
+  public synchronized void resume() {
+assert isSuspended() : this + " expected suspended state, got " + state;
+suspended = false;
+  }
+
   public synchronized RemoteProcedureException getException() {
 return exception;
   }
@@ -398,7 +421,7 @@ public abstract class Procedure implements 
Comparable {
* @return the timeout in msec
*/
   public int getTimeout() {
-return timeout;
+return timeout.intValue();
   }
 
   /**
@@ -494,7 +517,7 @@ public abstract class Procedure implements 
Comparable {
*/
   @InterfaceAudience.Private
   protected Procedure[] doExecute(final TEnvironment env)
-  throws ProcedureYieldException, InterruptedException {
+  throws ProcedureYieldException, ProcedureSuspendedException, 
InterruptedException {
 try {
   updateTimestamp();
   return execute(env);

http://git-wip-us.apache.org/repos/asf/hbase/blob/d5d9b7d5/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
--
diff --git 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
 
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index f43b65f..9d71f65 100644
--- 
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ 

[18/50] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()

[syuanjiang]

HBASE-15927 Remove HMaster.assignRegion()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0c159b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0c159b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0c159b5

Branch: refs/heads/hbase-12439
Commit: f0c159b5fed732cdc08bd2fd924ae81677d66a30
Parents: 0cbce07
Author: Matteo Bertozzi 
Authored: Fri Jun 3 12:36:25 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 3 12:36:25 2016 -0700

--
 .../org/apache/hadoop/hbase/master/HMaster.java |  4 --
 .../hadoop/hbase/HBaseTestingUtility.java   | 15 +
 .../hbase/client/TestMetaWithReplicas.java  |  2 +-
 .../master/TestAssignmentManagerOnCluster.java  | 61 
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   | 20 +++
 .../hadoop/hbase/util/TestHBaseFsckTwoRS.java   | 23 +++-
 6 files changed, 58 insertions(+), 67 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f8d0003..d368ffb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2378,10 +2378,6 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 return this.initializationBeforeMetaAssignment;
   }
 
-  public void assignRegion(HRegionInfo hri) {
-assignmentManager.assign(hri);
-  }
-
   /**
* Compute the average load across all region servers.
* Currently, this uses a very naive computation - just uses the number of

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 7b35815..bfa14cb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3200,6 +3200,21 @@ public class HBaseTestingUtility extends 
HBaseCommonTestingUtility {
   }
 
   /**
+   * Uses directly the assignment manager to assign the region.
+   * and waits until the specified region has completed assignment.
+   * @param tableName the table name
+   * @throws IOException
+   * @throw InterruptedException
+   * @return true if the region is assigned false otherwise.
+   */
+  public boolean assignRegion(final HRegionInfo regionInfo)
+  throws IOException, InterruptedException {
+final AssignmentManager am = 
getHBaseCluster().getMaster().getAssignmentManager();
+am.assign(regionInfo);
+return am.waitForAssignment(regionInfo);
+  }
+
+  /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index 5e302d2..8e87ceb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -448,7 +448,7 @@ public class TestMetaWithReplicas {
 // create in-memory state otherwise master won't assign
 TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
  .getRegionStates().createRegionState(h);
-TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h);
+TEST_UTIL.assignRegion(h);
 HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h);
 // check that problem exists
 HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
--
diff --git 

[03/50] hbase git commit: Remove the hbasecon banner logo

[syuanjiang]

Remove the hbasecon banner logo


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c80e2326
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c80e2326
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c80e2326

Branch: refs/heads/hbase-12439
Commit: c80e23264204c651ec7bd3e0fbf294ac728359e6
Parents: 5ea2f09
Author: stack 
Authored: Tue May 31 10:14:40 2016 -0700
Committer: stack 
Committed: Tue May 31 10:14:40 2016 -0700

--
 src/main/site/site.xml | 5 -
 1 file changed, 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c80e2326/src/main/site/site.xml
--
diff --git a/src/main/site/site.xml b/src/main/site/site.xml
index 6d4de53..fb237bb 100644
--- a/src/main/site/site.xml
+++ b/src/main/site/site.xml
@@ -43,13 +43,8 @@
 
   
   
-hbasecon2016
-
-images/hbasecon2016-stacked.png
-http://hbasecon.com/
   
   
 Apache HBase

[30/50] hbase git commit: HBASE-15957 RpcClientImpl.close never ends in some circumstances

[syuanjiang]

HBASE-15957 RpcClientImpl.close never ends in some circumstances

Signed-off-by: Enis Soztutar 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da88b482
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da88b482
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da88b482

Branch: refs/heads/hbase-12439
Commit: da88b4824054f57fbcbc7795469ab2369a39b5ed
Parents: 376ad0d
Author: Sergey Soldatov 
Authored: Sun Jun 5 23:46:03 2016 -0700
Committer: Enis Soztutar 
Committed: Tue Jun 7 11:33:03 2016 -0700

--
 .../apache/hadoop/hbase/ipc/RpcClientImpl.java  |  5 ++-
 .../hbase/ipc/IntegrationTestRpcClient.java | 35 
 2 files changed, 31 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/da88b482/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index d8c87e9..dc05af1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -1202,9 +1202,8 @@ public class RpcClientImpl extends AbstractRpcClient {
 }
 if (connsToClose != null) {
   for (Connection conn : connsToClose) {
-if (conn.markClosed(new InterruptedIOException("RpcClient is 
closing"))) {
-  conn.close();
-}
+conn.markClosed(new InterruptedIOException("RpcClient is closing"));
+conn.close();
   }
 }
 // wait until all connections are closed

http://git-wip-us.apache.org/repos/asf/hbase/blob/da88b482/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java
--
diff --git 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java
 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java
index c28f3e6..6c0fbcc 100644
--- 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java
+++ 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/ipc/IntegrationTestRpcClient.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hbase.ipc;
 
+import static org.apache.hadoop.hbase.ipc.RpcClient.SPECIFIC_WRITE_THREAD;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -41,12 +42,6 @@ import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.codec.Codec;
-import org.apache.hadoop.hbase.ipc.AbstractRpcClient;
-import org.apache.hadoop.hbase.ipc.AsyncRpcClient;
-import org.apache.hadoop.hbase.ipc.FifoRpcScheduler;
-import org.apache.hadoop.hbase.ipc.RpcClientImpl;
-import org.apache.hadoop.hbase.ipc.RpcScheduler;
-import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos;
 import 
org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoRequestProto;
 import 
org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoResponseProto;
@@ -290,6 +285,7 @@ public class IntegrationTestRpcClient {
   static class SimpleClient extends Thread {
 AbstractRpcClient rpcClient;
 AtomicBoolean running = new  AtomicBoolean(true);
+AtomicBoolean sending = new AtomicBoolean(false);
 AtomicReference exception = new AtomicReference<>(null);
 Cluster cluster;
 String id;
@@ -319,6 +315,7 @@ public class IntegrationTestRpcClient {
   if (address == null) {
 throw new IOException("Listener channel is closed");
   }
+  sending.set(true);
   ret = (EchoResponseProto)
   rpcClient.callBlockingMethod(md, null, param, ret, user, 
address);
 } catch (Exception e) {
@@ -340,6 +337,9 @@ public class IntegrationTestRpcClient {
 void stopRunning() {
   running.set(false);
 }
+boolean isSending() {
+  return sending.get();
+}
 
 void rethrowException() throws Throwable {
   if (exception.get() != null) {
@@ -348,6 +348,29 @@ public class IntegrationTestRpcClient {
 }
   }
 
+  /*
+  Test that not started connections are successfully removed from connection 
pool when
+  rpc client is closing.
+   */
+  @Test (timeout = 3)
+  public void testRpcWithWriteThread() throws IOException, 
InterruptedException {
+LOG.info("Starting test");
+Cluster cluster = new Cluster(1, 1);
+cluster.startServer();
+ 

[02/50] hbase git commit: HBASE-15919 Modify docs to change from @Rule to @ClassRule. Also clarify that timeout limits are on test case level. (Apekshit)

[syuanjiang]

HBASE-15919 Modify docs to change from @Rule to @ClassRule. Also clarify that 
timeout limits are on test case level. (Apekshit)

Change-Id: Ifcd0264ea147bcb1100db74d92da95b643f4793f

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ea2f092
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ea2f092
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ea2f092

Branch: refs/heads/hbase-12439
Commit: 5ea2f092332515eea48136d7d92f7b8ea72df15b
Parents: 75c2360
Author: Apekshit 
Authored: Tue May 31 03:30:50 2016 -0700
Committer: stack 
Committed: Tue May 31 10:12:00 2016 -0700

--
 src/main/asciidoc/_chapters/developer.adoc | 93 +
 1 file changed, 33 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5ea2f092/src/main/asciidoc/_chapters/developer.adoc
--
diff --git a/src/main/asciidoc/_chapters/developer.adoc 
b/src/main/asciidoc/_chapters/developer.adoc
index a11a04e..74ce3df 100644
--- a/src/main/asciidoc/_chapters/developer.adoc
+++ b/src/main/asciidoc/_chapters/developer.adoc
@@ -864,7 +864,8 @@ Also, keep in mind that if you are running tests in the 
`hbase-server` module yo
 [[hbase.unittests]]
 === Unit Tests
 
-Apache HBase unit tests are subdivided into four categories: small, medium, 
large, and integration with corresponding JUnit 
link:http://www.junit.org/node/581[categories]: `SmallTests`, `MediumTests`, 
`LargeTests`, `IntegrationTests`.
+Apache HBase test cases are subdivided into four categories: small, medium, 
large, and
+integration with corresponding JUnit 
link:http://www.junit.org/node/581[categories]: `SmallTests`, `MediumTests`, 
`LargeTests`, `IntegrationTests`.
 JUnit categories are denoted using java annotations and look like this in your 
unit test code.
 
 [source,java]
@@ -879,10 +880,11 @@ public class TestHRegionInfo {
 }
 
 
-The above example shows how to mark a unit test as belonging to the `small` 
category.
-All unit tests in HBase have a categorization.
+The above example shows how to mark a test case as belonging to the `small` 
category.
+All test cases in HBase should have a categorization.
 
-The first three categories, `small`, `medium`, and `large`, are for tests run 
when you type `$ mvn test`.
+The first three categories, `small`, `medium`, and `large`, are for test cases 
which run when you
+type `$ mvn test`.
 In other words, these three categorizations are for HBase unit tests.
 The `integration` category is not for unit tests, but for integration tests.
 These are run when you invoke `$ mvn verify`.
@@ -890,22 +892,23 @@ Integration tests are described in 
<>.
 
 HBase uses a patched maven surefire plugin and maven profiles to implement its 
unit test characterizations.
 
-Keep reading to figure which annotation of the set small, medium, and large to 
put on your new HBase unit test.
+Keep reading to figure which annotation of the set small, medium, and large to 
put on your new
+HBase test case.
 
 .Categorizing Tests
 Small Tests (((SmallTests)))::
-  _Small_ tests are executed in a shared JVM.
-  We put in this category all the tests that can be executed quickly in a 
shared JVM.
-  The maximum execution time for a small test is 15 seconds, and small tests 
should not use a (mini)cluster.
+  _Small_ test cases are executed in a shared JVM and individual test cases 
should run in 15 seconds
+   or less; i.e. a link:https://en.wikipedia.org/wiki/JUnit[junit test 
fixture], a java object made
+   up of test methods, should finish in under 15 seconds. These test cases can 
not use mini cluster.
+   These are run as part of patch pre-commit.
 
 Medium Tests (((MediumTests)))::
-  _Medium_ tests represent tests that must be executed before proposing a 
patch.
-  They are designed to run in less than 30 minutes altogether, and are quite 
stable in their results.
-  They are designed to last less than 50 seconds individually.
-  They can use a cluster, and each of them is executed in a separate JVM.
+  _Medium_ test cases are executed in separate JVM and individual test case 
should run in 50 seconds
+   or less. Together, they should take less than 30 minutes, and are quite 
stable in their results.
+   These test cases can use a mini cluster. These are run as part of patch 
pre-commit.
 
 Large Tests (((LargeTests)))::
-  _Large_ tests are everything else.
+  _Large_ test cases are everything else.
   They are typically large-scale tests, regression tests for specific bugs, 
timeout tests, performance tests.
   They are executed before a commit on the pre-integration machines.
   They can be run on the 

[17/50] hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication

[syuanjiang]

HBASE-15888 Extend HBASE-12769 for bulk load data replication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cbce076
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cbce076
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cbce076

Branch: refs/heads/hbase-12439
Commit: 0cbce07626b77d9aa75a16f5e52c19428865dce7
Parents: 72d3f2a
Author: Ashish Singhi 
Authored: Fri Jun 3 18:42:00 2016 +0530
Committer: Ashish Singhi 
Committed: Fri Jun 3 18:42:00 2016 +0530

--
 .../replication/ReplicationPeersZKImpl.java |  6 ++
 .../hbase/util/hbck/ReplicationChecker.java | 59 ++--
 2 files changed, 61 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 09d2100..15265d9 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -550,6 +550,12 @@ public class ReplicationPeersZKImpl extends 
ReplicationStateZKBase implements Re
   }
 }
   }
+  // Check for hfile-refs queue
+  if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+  && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+  }
 } catch (KeeperException e) {
   throw new ReplicationException("Could not check queues deleted with id=" 
+ peerId, e);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 89f2557..e472558 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
 @InterfaceAudience.Private
 public class ReplicationChecker {
   private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+  private final ZooKeeperWatcher zkw;
   private final ErrorReporter errorReporter;
   private final ReplicationQueuesClient queuesClient;
   private final ReplicationPeers replicationPeers;
   private final ReplicationQueueDeletor queueDeletor;
   // replicator with its queueIds for removed peers
   private final Map undeletedQueueIds = new HashMap<>();
-  
+  // replicator with its undeleted queueIds for removed peers in hfile-refs 
queue
+  private Set undeletedHFileRefsQueueIds = new HashSet<>();
+  private final String hfileRefsZNode;
+
   public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw, 
ClusterConnection connection,
   ErrorReporter errorReporter) throws IOException {
 try {
+  this.zkw = zkw;
   this.errorReporter = errorReporter;
   this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw, 
conf, connection);
   this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
 } catch (ReplicationException e) {
   throw new IOException("failed to construct ReplicationChecker", e);
 }
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication", 
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode, 
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+  
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
   }
 
   public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
 } catch (KeeperException ke) {
   throw new IOException(ke);
 }
+
+checkUnDeletedHFileRefsQueues(peerIds);
+  }
+
+  private void checkUnDeletedHFileRefsQueues(Set peerIds) throws 
IOException {
+try {
+  if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+return;
+  }
+   

[25/50] hbase git commit: HBASE-15174 Client Public API should not have PB objects in 2.0 (Ram)

[syuanjiang]

HBASE-15174 Client Public API should not have PB objects in 2.0 (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b21c56e7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b21c56e7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b21c56e7

Branch: refs/heads/hbase-12439
Commit: b21c56e7958652ca6e6daf04642eb51abaf2b3d7
Parents: 70762fa
Author: Ramkrishna 
Authored: Mon Jun 6 10:11:38 2016 +0530
Committer: Ramkrishna 
Committed: Mon Jun 6 10:11:38 2016 +0530

--
 .../org/apache/hadoop/hbase/RegionLoad.java |   1 +
 .../org/apache/hadoop/hbase/ServerLoad.java |   2 +
 .../hadoop/hbase/client/RegionLoadStats.java|   4 +-
 .../hbase/TestInterfaceAudienceAnnotations.java | 152 +++
 .../org/apache/hadoop/hbase/util/Triple.java|   4 +
 5 files changed, 161 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
--
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index 5bf2ec7..b5852d4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -38,6 +38,7 @@ public class RegionLoad {
 
   protected ClusterStatusProtos.RegionLoad regionLoadPB;
 
+  @InterfaceAudience.Private
   public RegionLoad(ClusterStatusProtos.RegionLoad regionLoadPB) {
 this.regionLoadPB = regionLoadPB;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
--
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
index 1ddcc20..3ea59db 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
@@ -57,6 +57,7 @@ public class ServerLoad {
   private long totalCompactingKVs = 0;
   private long currentCompactedKVs = 0;
 
+  @InterfaceAudience.Private
   public ServerLoad(ClusterStatusProtos.ServerLoad serverLoad) {
 this.serverLoad = serverLoad;
 for (ClusterStatusProtos.RegionLoad rl: serverLoad.getRegionLoadsList()) {
@@ -81,6 +82,7 @@ public class ServerLoad {
   // NOTE: Function name cannot start with "get" because then an 
OpenDataException is thrown because
   // HBaseProtos.ServerLoad cannot be converted to an open data type(see 
HBASE-5967).
   /* @return the underlying ServerLoad protobuf object */
+  @InterfaceAudience.Private
   public ClusterStatusProtos.ServerLoad obtainServerLoadPB() {
 return serverLoad;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
index 443026f..bfdb216 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.hbase.client;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
 /**
  * POJO representing region server load
  */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class RegionLoadStats {
   int memstoreLoad;
   int heapOccupancy;

http://git-wip-us.apache.org/repos/asf/hbase/blob/b21c56e7/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java
 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java
index 0e0fbb0..426b6a7 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestInterfaceAudienceAnnotations.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.hbase;
 
 import java.io.IOException;
 import java.lang.annotation.Annotation;
+import java.lang.reflect.Constructor;

[11/50] hbase git commit: HBASE-15944 Spark test flooding mvn output. Redirect test logs to file. This doesn't fix the problem fully as I still see few logs being dumped in stdout. But it cleans up ma

[syuanjiang]

HBASE-15944 Spark test flooding mvn output. Redirect test logs to file. This 
doesn't fix the problem fully as I still see few logs being dumped in stdout. 
But it cleans up majority of the earlier dump. (Apekshit)

Change-Id: I6893301d154078a7cfb6b9af2eedc744deafb8d7

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4ffea771
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4ffea771
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4ffea771

Branch: refs/heads/hbase-12439
Commit: 4ffea7711acc33dc3c0a75479b510d0683c11523
Parents: cfe868d
Author: Apekshit 
Authored: Thu Jun 2 01:43:35 2016 -0700
Committer: stack 
Committed: Thu Jun 2 08:44:08 2016 -0700

--
 hbase-spark/src/test/resources/log4j.properties | 10 +-
 1 file changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4ffea771/hbase-spark/src/test/resources/log4j.properties
--
diff --git a/hbase-spark/src/test/resources/log4j.properties 
b/hbase-spark/src/test/resources/log4j.properties
index 4eeeb2c..6d9415b 100644
--- a/hbase-spark/src/test/resources/log4j.properties
+++ b/hbase-spark/src/test/resources/log4j.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Define some default values that can be overridden by system properties
-hbase.root.logger=INFO,console
+hbase.root.logger=INFO,FA
 hbase.log.dir=.
 hbase.log.file=hbase.log
 
@@ -50,6 +50,14 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] 
%C{2}(%L): %m%n
 
+#File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/log-output.txt
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): 
%m%n
+log4j.appender.FA.Threshold = INFO
+
 # Custom Logging levels
 
 #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG

[09/50] hbase git commit: HBASE-15858 Some region server group shell commands don't work

[syuanjiang]

HBASE-15858 Some region server group shell commands don't work


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a0f49c98
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a0f49c98
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a0f49c98

Branch: refs/heads/hbase-12439
Commit: a0f49c988419d48f6c655f46ac78f8199c643b50
Parents: 53eb27b
Author: tedyu 
Authored: Wed Jun 1 21:13:33 2016 -0700
Committer: tedyu 
Committed: Wed Jun 1 21:13:33 2016 -0700

--
 hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb  |  2 +-
 .../src/main/ruby/shell/commands/get_server_rsgroup.rb   |  2 +-
 hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb| 11 +++
 3 files changed, 9 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb 
b/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb
index 6772aa1..ce4be71 100644
--- a/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/get_rsgroup.rb
@@ -34,7 +34,7 @@ EOF
   def command(group_name)
 now = Time.now
 formatter.header(['GROUP INFORMATION'])
-group_admin.get_rsgroup(group_name) do |s|
+rsgroup_admin.get_rsgroup(group_name) do |s|
   formatter.row([s])
 end
 formatter.footer(now)

http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb 
b/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb
index 322f6bb..a689a7c 100644
--- a/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/get_server_rsgroup.rb
@@ -31,7 +31,7 @@ EOF
 
   def command(server)
 now = Time.now
-group_name = rsgroup_admin.getGroupOfServer(server).getName
+group_name = rsgroup_admin.getRSGroupOfServer(server).getName
 formatter.row([group_name])
 formatter.footer(now, 1)
   end

http://git-wip-us.apache.org/repos/asf/hbase/blob/a0f49c98/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb
--
diff --git a/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb 
b/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb
index d892775..1040ed8 100644
--- a/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb
+++ b/hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb
@@ -49,12 +49,15 @@ module Hbase
   assert_not_nil(group)
   assert_equal(0, group.getServers.count)
 
-  hostport =
-  
@rsgroup_admin.getRSGroupInfo('default').getServers.iterator.next.toString
+  hostport = 
@rsgroup_admin.getRSGroupInfo('default').getServers.iterator.next
+  @shell.command('get_rsgroup', 'default')
+  hostPortStr = hostport.toString
+  @shell.command('get_server_rsgroup', [hostPortStr])
   @shell.command('move_rsgroup_servers',
  group_name,
- [hostport])
+ [hostPortStr])
   assert_equal(1, 
@rsgroup_admin.getRSGroupInfo(group_name).getServers.count)
+  assert_equal(group_name, 
@rsgroup_admin.getRSGroupOfServer(hostport).getName)
 
   @shell.command('move_rsgroup_tables',
  group_name,
@@ -65,7 +68,7 @@ module Hbase
   @hbase.rsgroup_admin(@formatter).get_rsgroup(group_name) do |line|
 case count
 when 1
-  assert_equal(hostport, line)
+  assert_equal(hostPortStr, line)
 when 3
   assert_equal(table_name, line)
 end

[27/50] hbase git commit: HBASE-15965 - Testing by executing a command will cover the exact path users will trigger, so its better then directly calling library functions in tests. Changing the tests

[syuanjiang]

HBASE-15965
- Testing by executing a command will cover the exact path users will trigger, 
so its better then directly calling library functions in tests. Changing the 
tests to use @shell.command(:, args) to execute them like it's a 
command coming from shell.

Norm change:
Commands should print the output user would like to see, but in the end, should 
also return the relevant value. This way:
- Tests can use returned value to check that functionality works
- Tests can capture stdout to assert particular kind of output user should see.
- We do not print the return value in interactive mode and keep the output 
clean. See Shell.command() function.

Bugs found due to this change:
- Uncovered bug in major_compact.rb with this approach. It was calling 
admin.majorCompact() which doesn't exist but our tests didn't catch it since 
they directly tested admin.major_compact()
- Enabled TestReplicationShell. If it's bad, flaky infra will take care of it.

Change-Id: I5d8af16bf477a79a2f526a5bf11c245b02b7d276


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15c03fd1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15c03fd1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15c03fd1

Branch: refs/heads/hbase-12439
Commit: 15c03fd1c97c271aca6dc30feab35ec0c9f8edbe
Parents: 3d7840a
Author: Apekshit Sharma 
Authored: Mon Jun 6 13:35:06 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Jun 6 17:50:22 2016 -0700

--
 .../replication/ReplicationPeerConfig.java  |   6 +-
 hbase-shell/src/main/ruby/hbase/admin.rb|  23 +-
 hbase-shell/src/main/ruby/hbase/table.rb|   1 +
 hbase-shell/src/main/ruby/shell.rb  |  12 +-
 hbase-shell/src/main/ruby/shell/commands.rb |   2 +
 .../main/ruby/shell/commands/balance_rsgroup.rb |   9 +-
 .../src/main/ruby/shell/commands/create.rb  |   2 +
 .../src/main/ruby/shell/commands/exists.rb  |   4 +-
 .../src/main/ruby/shell/commands/get_auths.rb   |   1 +
 .../main/ruby/shell/commands/get_peer_config.rb |   1 +
 .../src/main/ruby/shell/commands/is_enabled.rb  |   4 +-
 .../shell/commands/list_namespace_tables.rb |   1 +
 .../ruby/shell/commands/list_peer_configs.rb|   1 +
 .../src/main/ruby/shell/commands/list_peers.rb  |   1 +
 .../main/ruby/shell/commands/locate_region.rb   |   1 +
 .../main/ruby/shell/commands/major_compact.rb   |   2 +-
 .../ruby/shell/commands/show_peer_tableCFs.rb   |   4 +-
 .../src/main/ruby/shell/commands/truncate.rb|   3 +-
 .../ruby/shell/commands/truncate_preserve.rb|   3 +-
 .../hbase/client/TestReplicationShell.java  |   2 +-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   | 229 +--
 .../test/ruby/hbase/replication_admin_test.rb   | 110 -
 .../ruby/hbase/visibility_labels_admin_test.rb  |  20 +-
 hbase-shell/src/test/ruby/test_helper.rb|  23 +-
 24 files changed, 254 insertions(+), 211 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/15c03fd1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index 7799de6..1d2066c 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -96,8 +96,10 @@ public class ReplicationPeerConfig {
   @Override
   public String toString() {
 StringBuilder builder = new 
StringBuilder("clusterKey=").append(clusterKey).append(",");
-
builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(",")
-.append("tableCFs=").append(tableCFsMap.toString());
+
builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(",");
+if (tableCFsMap != null) {
+  builder.append("tableCFs=").append(tableCFsMap.toString());
+}
 return builder.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/15c03fd1/hbase-shell/src/main/ruby/hbase/admin.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index f32376d..d66c1d6 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -458,16 +458,17 @@ module Hbase
 
 
#--
 # Truncates table (deletes all records by recreating the table)
-def 

[29/50] hbase git commit: HBASE-15698 Increment TimeRange not serialized to server (Ted Yu)

[syuanjiang]

HBASE-15698 Increment TimeRange not serialized to server (Ted Yu)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/376ad0d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/376ad0d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/376ad0d9

Branch: refs/heads/hbase-12439
Commit: 376ad0d9868e563e4153f8725afbf00a0293f72c
Parents: 7fd3532
Author: Andrew Purtell 
Authored: Mon Jun 6 16:59:43 2016 -0700
Committer: Andrew Purtell 
Committed: Mon Jun 6 22:17:00 2016 -0700

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  21 +-
 .../coprocessor/TestIncrementTimeRange.java | 196 +++
 2 files changed, 210 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/376ad0d9/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 0c34a17..fecc3c2 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -,6 +,16 @@ public final class ProtobufUtil {
 return builder.build();
   }
 
+  static void setTimeRange(final MutationProto.Builder builder, final 
TimeRange timeRange) {
+if (!timeRange.isAllTime()) {
+  HBaseProtos.TimeRange.Builder timeRangeBuilder =
+HBaseProtos.TimeRange.newBuilder();
+  timeRangeBuilder.setFrom(timeRange.getMin());
+  timeRangeBuilder.setTo(timeRange.getMax());
+  builder.setTimeRange(timeRangeBuilder.build());
+}
+  }
+
   /**
* Convert a client Increment to a protobuf Mutate.
*
@@ -1126,13 +1136,7 @@ public final class ProtobufUtil {
   builder.setNonce(nonce);
 }
 TimeRange timeRange = increment.getTimeRange();
-if (!timeRange.isAllTime()) {
-  HBaseProtos.TimeRange.Builder timeRangeBuilder =
-HBaseProtos.TimeRange.newBuilder();
-  timeRangeBuilder.setFrom(timeRange.getMin());
-  timeRangeBuilder.setTo(timeRange.getMax());
-  builder.setTimeRange(timeRangeBuilder.build());
-}
+setTimeRange(builder, timeRange);
 ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
 QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
 for (Map.Entry family: 
increment.getFamilyCellMap().entrySet()) {
@@ -1253,6 +1257,9 @@ public final class ProtobufUtil {
   final MutationProto.Builder builder, long nonce) throws IOException {
 getMutationBuilderAndSetCommonFields(type, mutation, builder);
 builder.setAssociatedCellCount(mutation.size());
+if (mutation instanceof Increment) {
+  setTimeRange(builder, ((Increment)mutation).getTimeRange());
+}
 if (nonce != HConstants.NO_NONCE) {
   builder.setNonce(nonce);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/376ad0d9/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
new file mode 100644
index 000..35ed531
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestIncrementTimeRange.java
@@ -0,0 +1,196 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+

[12/50] hbase git commit: HBASE-15938 submit-patch.py: Don't crash if there are tests with same name. Refactor: Split out flaky dashboard html template to separate file. (Apekshit)

[syuanjiang]

HBASE-15938 submit-patch.py: Don't crash if there are tests with same name. 
Refactor: Split out flaky dashboard html template to separate file. (Apekshit)

Change-Id: Ie5875bdefbf886984a57dfc85661be2ac9592a7b

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9593a9f3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9593a9f3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9593a9f3

Branch: refs/heads/hbase-12439
Commit: 9593a9f39663e454a24f16c907e00e19cb65b903
Parents: 4ffea77
Author: Apekshit 
Authored: Wed Jun 1 19:12:50 2016 -0700
Committer: stack 
Committed: Thu Jun 2 08:55:42 2016 -0700

--
 dev-support/findHangingTests.py   |  14 ++-
 dev-support/flaky-dashboard-template.html | 122 +
 dev-support/report-flakies.py | 112 +--
 3 files changed, 137 insertions(+), 111 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/findHangingTests.py
--
diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py
index 9ef8708..28f4895 100755
--- a/dev-support/findHangingTests.py
+++ b/dev-support/findHangingTests.py
@@ -46,14 +46,22 @@ def get_bad_tests(console_url):
 result1 = re.match("^Running org.apache.hadoop.hbase.(\w*\.)*(\w*)", 
line)
 if result1:
 test_case = result1.group(2)
-hanging_tests.add(test_case)
-all_tests.add(test_case)
+if test_case in all_tests:
+print  ("ERROR! Multiple tests with same name '{}'. Might get 
wrong results "
+   "for this test.".format(test_case))
+else:
+hanging_tests.add(test_case)
+all_tests.add(test_case)
 result2 = re.match("^Tests run:.*- in 
org.apache.hadoop.hbase.(\w*\.)*(\w*)", line)
 if result2:
 test_case = result2.group(2)
-hanging_tests.remove(test_case)
 if "FAILURE!" in line:
 failed_tests.add(test_case)
+if test_case not in hanging_tests:
+print  ("ERROR! No test '{}' found in hanging_tests. Might get 
wrong results "
+"for this test.".format(test_case))
+else:
+hanging_tests.remove(test_case)
 result3 = re.match("^\s+(\w*).*\sTestTimedOut", line)
 if result3:
 test_case = result3.group(1)

http://git-wip-us.apache.org/repos/asf/hbase/blob/9593a9f3/dev-support/flaky-dashboard-template.html
--
diff --git a/dev-support/flaky-dashboard-template.html 
b/dev-support/flaky-dashboard-template.html
new file mode 100644
index 000..77dfc86
--- /dev/null
+++ b/dev-support/flaky-dashboard-template.html
@@ -0,0 +1,122 @@
+
+
+
+
+Apache HBase Flaky Dashboard
+
+table {
+table-layout: fixed;
+}
+th {
+font-size: 15px;
+}
+td {
+font-size: 18px;
+vertical-align: text-top;
+overflow: hidden;
+white-space: nowrap;
+}
+.show_hide_button {
+font-size: 100%;
+padding: .5em 1em;
+border: 0 rgba(0,0,0,0);
+border-radius: 10px;
+}
+
+
+
+
+https://hbase.apache.org/images/hbase_logo_with_orca_large.png;>
+
+  
+  Apache HBase Flaky Tests Dashboard
+  
+
+
+{% set counter = 0 %}
+{% for url in results %}
+{% set result = results[url] %}
+{# Dedup ids since test names may duplicate across urls #}
+{% set counter = counter + 1 %}
+Job : {{ url 
|e }}
+
+
+
+
+Test Name
+Flakyness
+Failed/Timeout/Hanging
+Run Ids
+
+{% for test in result %}
+{% set all = result[test]['all'] %}
+{% set failed = result[test]['failed'] %}
+{% set timeout = result[test]['timeout'] %}
+{% set hanging = result[test]['hanging'] %}
+{% set success = all.difference(failed).difference(hanging) %}
+
+{{ test |e }}
+{% set flakyness =
+(failed|length + hanging|length) * 100 / all|length %}
+{% if flakyness == 100 %}
+
+{% else %}
+
+{% endif %}
+{{ "{:.1f}% ({} / {})".format(
+flakyness, failed|length + hanging|length, all|length) }}
+
+
+{{ failed|length }} / {{ timeout|length }} / {{ hanging|length }}
+
+
+{% set id = "details_" ~ test ~ "_" ~ counter  %}
+   

[04/50] hbase git commit: HBASE-15917 Addendum. Fix bug in report-flakies.py where hanging tests are not being added to flaky list. (Apekshit) ADDENDUM #2!

[syuanjiang]

HBASE-15917 Addendum. Fix bug in report-flakies.py where hanging tests are not 
being added to flaky list. (Apekshit)
ADDENDUM #2!

Change-Id: I9c55932d0f9e65b72ec8d3ae714144536b2bfe0a

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb64cd9d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb64cd9d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb64cd9d

Branch: refs/heads/hbase-12439
Commit: eb64cd9dd13ba297539c409989c63e800cb378a1
Parents: c80e232
Author: Apekshit 
Authored: Tue May 31 02:29:40 2016 -0700
Committer: stack 
Committed: Tue May 31 10:16:40 2016 -0700

--
 dev-support/report-flakies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/eb64cd9d/dev-support/report-flakies.py
--
diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py
index bdc88dc..c0d16c7 100755
--- a/dev-support/report-flakies.py
+++ b/dev-support/report-flakies.py
@@ -161,7 +161,7 @@ for url_max_build in expanded_urls:
 print ""
 
 
-all_bad_tests = all_timeout_tests.union(all_failed_tests)
+all_bad_tests = all_hanging_tests.union(all_failed_tests)
 if args.mvn:
 includes = ",".join(all_bad_tests)
 with open("./includes", "w") as inc_file:

[07/50] hbase git commit: HBASE-15932 Shell test fails due to uninitialized constant

[syuanjiang]

HBASE-15932 Shell test fails due to uninitialized constant


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbb95cd3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbb95cd3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbb95cd3

Branch: refs/heads/hbase-12439
Commit: cbb95cd3a9bf9a9f8558560ae58f4061a73f15a8
Parents: 015f2ef
Author: tedyu 
Authored: Wed Jun 1 10:35:00 2016 -0700
Committer: tedyu 
Committed: Wed Jun 1 10:35:00 2016 -0700

--
 .../src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java   | 2 +-
 hbase-shell/src/main/ruby/hbase.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cbb95cd3/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 799bf0b..b75e8cd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -64,7 +64,7 @@ public class HColumnDescriptor implements 
Comparable {
   // Version 11 -- add column family level configuration.
   private static final byte COLUMN_DESCRIPTOR_VERSION = (byte) 11;
 
-  private static final String IN_MEMORY_COMPACTION = "IN_MEMORY_COMPACTION";
+  public static final String IN_MEMORY_COMPACTION = "IN_MEMORY_COMPACTION";
 
   // These constants are used as FileInfo keys
   public static final String COMPRESSION = "COMPRESSION";

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbb95cd3/hbase-shell/src/main/ruby/hbase.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase.rb 
b/hbase-shell/src/main/ruby/hbase.rb
index 189fa3a..bc6f37c 100644
--- a/hbase-shell/src/main/ruby/hbase.rb
+++ b/hbase-shell/src/main/ruby/hbase.rb
@@ -39,7 +39,7 @@ module HBaseConstants
   NAME = org.apache.hadoop.hbase.HConstants::NAME
   VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS
   IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY
-  IN_MEMORY_COMPACTION = 
org.apache.hadoop.hbase.HConstants::IN_MEMORY_COMPACTION
+  IN_MEMORY_COMPACTION = 
org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY_COMPACTION
   METADATA = org.apache.hadoop.hbase.HConstants::METADATA
   STOPROW = "STOPROW"
   STARTROW = "STARTROW"

[06/50] hbase git commit: HBASE-15923 Shell rows counter test fails

[syuanjiang]

HBASE-15923 Shell rows counter test fails


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/015f2ef6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/015f2ef6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/015f2ef6

Branch: refs/heads/hbase-12439
Commit: 015f2ef6292df52270df8845ccd244a97deb9c98
Parents: 73ec338
Author: tedyu 
Authored: Tue May 31 14:21:32 2016 -0700
Committer: tedyu 
Committed: Tue May 31 14:21:32 2016 -0700

--
 hbase-shell/src/test/ruby/hbase/table_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/015f2ef6/hbase-shell/src/test/ruby/hbase/table_test.rb
--
diff --git a/hbase-shell/src/test/ruby/hbase/table_test.rb 
b/hbase-shell/src/test/ruby/hbase/table_test.rb
index d4547b7..a617bc5 100644
--- a/hbase-shell/src/test/ruby/hbase/table_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/table_test.rb
@@ -561,7 +561,7 @@ module Hbase
 define_test "scan with a block should yield rows and return rows counter" 
do
   rows = {}
   res = @test_table._scan_internal { |row, cells| rows[row] = cells }
-  assert_equal(rows.keys.size, res)
+  assert_equal([rows.keys.size,false], res)
 end
 
 define_test "scan should support COLUMNS with value CONVERTER information" 
do

[21/50] hbase git commit: HBASE-15929 There are two tests named TestRegionServerMetrics. This causes slight issues when parsing console output for hanging tests and flaky tests analysis. Moving .../hb

[syuanjiang]

HBASE-15929 There are two tests named TestRegionServerMetrics. This causes 
slight issues when parsing console output for hanging tests and flaky tests 
analysis. Moving .../hbase/TestRegionServerMetrics.java to 
.../hbase/regionserver/TestRegionServerReadRequestMetrics.java. (Apekshit)

Change-Id: I379c15fe2c2c01bed53bddf7619d5f2a07c5640e


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a53d8b3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a53d8b3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a53d8b3

Branch: refs/heads/hbase-12439
Commit: 9a53d8b3850f0521c3a4ad194a3b84d7f4145007
Parents: b557f0b
Author: Apekshit 
Authored: Thu Jun 2 20:06:20 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 15:13:49 2016 -0700

--
 .../hadoop/hbase/TestRegionServerMetrics.java   | 379 --
 .../TestRegionServerReadRequestMetrics.java | 387 +++
 2 files changed, 387 insertions(+), 379 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a53d8b3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
deleted file mode 100644
index 76e5842..000
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-@Category(MediumTests.class)
-public class TestRegionServerMetrics {
-  private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
-  private static final TableName TABLE_NAME = TableName.valueOf("test");
-  private static final byte[] CF1 = "c1".getBytes();
-  private static final byte[] CF2 = "c2".getBytes();
-
-  private static final byte[] ROW1 = "a".getBytes();
-  private static final byte[] ROW2 = "b".getBytes();
-  private static final byte[] ROW3 = "c".getBytes();
-  private static final byte[] COL1 = "q1".getBytes();
-  private static final byte[] COL2 = "q2".getBytes();
-  private static final byte[] COL3 = "q3".getBytes();
-  private static final byte[] VAL1 = "v1".getBytes();
-  private static final byte[] VAL2 = "v2".getBytes();
-  private static final byte[] VAL3 = Bytes.toBytes(0L);
-
-  private static final int MAX_TRY = 20;
-  private static final int SLEEP_MS = 100;
-  private static final int TTL = 1;
-
-  private static Admin admin;
-  private static Collection serverNames;
-  private static Table table;
-  private static List tableRegions;
-
-  private static Map requestsMap = new HashMap<>();
-  private static 

[26/50] hbase git commit: HBASE-15954 REST server should log requests with TRACE instead of DEBUG

[syuanjiang]

HBASE-15954 REST server should log requests with TRACE instead of DEBUG


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3d7840a1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3d7840a1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3d7840a1

Branch: refs/heads/hbase-12439
Commit: 3d7840a173aab97fb72409fa8c0f161fd7ad0e8f
Parents: b21c56e
Author: Enis Soztutar 
Authored: Mon Jun 6 10:58:37 2016 -0700
Committer: Enis Soztutar 
Committed: Mon Jun 6 10:58:37 2016 -0700

--
 .../hadoop/hbase/rest/MultiRowResource.java |  4 +-
 .../hbase/rest/NamespacesInstanceResource.java  | 24 -
 .../hadoop/hbase/rest/NamespacesResource.java   |  4 +-
 .../hbase/rest/ProtobufStreamingUtil.java   | 10 ++--
 .../apache/hadoop/hbase/rest/RESTServer.java| 12 +++--
 .../apache/hadoop/hbase/rest/RESTServlet.java   |  5 +-
 .../hadoop/hbase/rest/RegionsResource.java  |  4 +-
 .../apache/hadoop/hbase/rest/RootResource.java  |  4 +-
 .../apache/hadoop/hbase/rest/RowResource.java   | 54 ++--
 .../hbase/rest/ScannerInstanceResource.java | 32 +++-
 .../hadoop/hbase/rest/ScannerResource.java  | 17 +++---
 .../hadoop/hbase/rest/SchemaResource.java   | 22 
 .../rest/StorageClusterStatusResource.java  |  4 +-
 .../rest/StorageClusterVersionResource.java |  4 +-
 .../apache/hadoop/hbase/rest/TableResource.java | 26 ++
 .../hadoop/hbase/rest/VersionResource.java  | 10 ++--
 .../apache/hadoop/hbase/rest/client/Client.java | 44 
 .../hadoop/hbase/rest/filter/AuthFilter.java|  4 +-
 .../rest/filter/RestCsrfPreventionFilter.java   | 15 +++---
 .../consumer/ProtobufMessageBodyConsumer.java   |  6 +--
 .../hadoop/hbase/util/ConnectionCache.java  |  6 ++-
 21 files changed, 169 insertions(+), 142 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3d7840a1/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index c88ac91..8ff3ef6 100644
--- 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -86,7 +86,9 @@ public class MultiRowResource extends ResourceBase implements 
Constants {
   }
   model.addRow(rowModel);
 } else {
-  LOG.trace("The row : " + rk + " not found in the table.");
+  if (LOG.isTraceEnabled()) {
+LOG.trace("The row : " + rk + " not found in the table.");
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/3d7840a1/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java
 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java
index 8f64738..c832905 100644
--- 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java
+++ 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java
@@ -91,8 +91,8 @@ public class NamespacesInstanceResource extends ResourceBase {
 MIMETYPE_PROTOBUF_IETF})
   public Response get(final @Context ServletContext context,
   final @Context UriInfo uriInfo) {
-if (LOG.isDebugEnabled()) {
-  LOG.debug("GET " + uriInfo.getAbsolutePath());
+if (LOG.isTraceEnabled()) {
+  LOG.trace("GET " + uriInfo.getAbsolutePath());
 }
 servlet.getMetrics().incrementRequests(1);
 
@@ -135,8 +135,8 @@ public class NamespacesInstanceResource extends 
ResourceBase {
   @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
 MIMETYPE_PROTOBUF_IETF})
   public Response put(final NamespacesInstanceModel model, final @Context 
UriInfo uriInfo) {
-if (LOG.isDebugEnabled()) {
-  LOG.debug("PUT " + uriInfo.getAbsolutePath());
+if (LOG.isTraceEnabled()) {
+  LOG.trace("PUT " + uriInfo.getAbsolutePath());
 }
 servlet.getMetrics().incrementRequests(1);
 return processUpdate(model, true, uriInfo);
@@ -151,8 +151,8 @@ public class NamespacesInstanceResource extends 
ResourceBase {
   @PUT
   public Response putNoBody(final byte[] message,
   final @Context UriInfo uriInfo, final @Context HttpHeaders headers) {
-if (LOG.isDebugEnabled()) {
-  LOG.debug("PUT " + uriInfo.getAbsolutePath());
+if (LOG.isTraceEnabled()) {
+  LOG.trace("PUT " 

[13/50] hbase git commit: HBASE-15933 Addendum - make merge decision when sizes of both regions are known

[syuanjiang]

HBASE-15933 Addendum - make merge decision when sizes of both regions are known


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd258800
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd258800
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd258800

Branch: refs/heads/hbase-12439
Commit: cd2588001cf31ad2fb2020f9e021c9b1be1b76fc
Parents: 9593a9f
Author: tedyu 
Authored: Thu Jun 2 09:30:36 2016 -0700
Committer: tedyu 
Committed: Thu Jun 2 09:30:36 2016 -0700

--
 .../hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cd258800/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index d209eb7..7a54d87 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -182,7 +182,7 @@ public class SimpleRegionNormalizer implements 
RegionNormalizer {
 if (mergeEnabled) {
   HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
   long regionSize2 = getRegionSize(hri2);
-  if (regionSize + regionSize2 < avgRegionSize) {
+  if (regionSize > 0 && regionSize2 > 0 && regionSize + regionSize2 < 
avgRegionSize) {
 LOG.info("Table " + table + ", small region size: " + regionSize
   + " plus its neighbor size: " + regionSize2
   + ", less than the avg size " + avgRegionSize + ", merging 
them");

[05/50] hbase git commit: HBASE-15907 updates for HBase Shell pre-splitting docs

[syuanjiang]

HBASE-15907 updates for HBase Shell pre-splitting docs

(cherry picked from commit 01adec574d9ccbdd6183466cb8ee6b43935d69ca)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73ec3385
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73ec3385
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73ec3385

Branch: refs/heads/hbase-12439
Commit: 73ec33856d0ee2ac1e058c6f7e1ccffa4476fbc0
Parents: eb64cd9
Author: Ronan Stokes 
Authored: Mon May 30 23:52:43 2016 -0700
Committer: Misty Stanley-Jones 
Committed: Tue May 31 13:52:46 2016 -0700

--
 src/main/asciidoc/_chapters/performance.adoc | 19 ++-
 src/main/asciidoc/_chapters/shell.adoc   | 62 +++
 2 files changed, 79 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/73ec3385/src/main/asciidoc/_chapters/performance.adoc
--
diff --git a/src/main/asciidoc/_chapters/performance.adoc 
b/src/main/asciidoc/_chapters/performance.adoc
index a0c00ae..5f27640 100644
--- a/src/main/asciidoc/_chapters/performance.adoc
+++ b/src/main/asciidoc/_chapters/performance.adoc
@@ -499,7 +499,7 @@ For bulk imports, this means that all clients will write to 
the same region unti
 A useful pattern to speed up the bulk import process is to pre-create empty 
regions.
 Be somewhat conservative in this, because too-many regions can actually 
degrade performance.
 
-There are two different approaches to pre-creating splits.
+There are two different approaches to pre-creating splits using the HBase API.
 The first approach is to rely on the default `Admin` strategy (which is 
implemented in `Bytes.split`)...
 
 [source,java]
@@ -511,7 +511,7 @@ int numberOfRegions = ...;  // # of regions to create
 admin.createTable(table, startKey, endKey, numberOfRegions);
 
 
-And the other approach is to define the splits yourself...
+And the other approach, using the HBase API, is to define the splits 
yourself...
 
 [source,java]
 
@@ -519,8 +519,23 @@ byte[][] splits = ...;   // create your own splits
 admin.createTable(table, splits);
 
 
+You can achieve a similar effect using the HBase Shell to create tables by 
specifying split options. 
+
+[source]
+
+# create table with specific split points
+hbase>create 't1','f1',SPLITS => ['\x10\x00', '\x20\x00', '\x30\x00', 
'\x40\x00']
+
+# create table with four regions based on random bytes keys
+hbase>create 't2','f1', { NUMREGIONS => 4 , SPLITALGO => 'UniformSplit' }
+
+# create table with five regions based on hex keys
+create 't3','f1', { NUMREGIONS => 5, SPLITALGO => 'HexStringSplit' }
+
+
 See <> for issues related to understanding your keyspace 
and pre-creating regions.
 See <>  
for discussion on manually pre-splitting regions.
+See <> for more details of using the HBase Shell to 
pre-split tables.
 
 [[def.log.flush]]
 ===  Table Creation: Deferred Log Flush

http://git-wip-us.apache.org/repos/asf/hbase/blob/73ec3385/src/main/asciidoc/_chapters/shell.adoc
--
diff --git a/src/main/asciidoc/_chapters/shell.adoc 
b/src/main/asciidoc/_chapters/shell.adoc
index a4237fd..8f1f59b 100644
--- a/src/main/asciidoc/_chapters/shell.adoc
+++ b/src/main/asciidoc/_chapters/shell.adoc
@@ -352,6 +352,68 @@ hbase(main):022:0> Date.new(1218920189000).toString() => 
"Sat Aug 16 20:56:29 UT
 
 To output in a format that is exactly like that of the HBase log format will 
take a little messing with 
link:http://download.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html[SimpleDateFormat].
 
+[[tricks.pre-split]]
+=== Pre-splitting tables with the HBase Shell
+You can use a variety of options to pre-split tables when creating them via 
the HBase Shell `create` command.
+
+The simplest approach is to specify an array of split points when creating the 
table. Note that when specifying string literals as split points, these will 
create split points based on the underlying byte representation of the string. 
So when specifying a split point of '10', we are actually specifying the byte 
split point '\x31\30'.
+
+The split points will define `n+1` regions where `n` is the number of split 
points. The lowest region will contain all keys from the lowest possible key up 
to but not including the first split point key.
+The next region will contain keys from the first split point up to, but not 
including the next split point key.
+This will continue for all split points up to the last. The last region will 
be defined from the last split point up to the maximum possible key.
+
+[source]
+
+hbase>create 't1','f',SPLITS 

[16/50] hbase git commit: HBASE-15939 Two shell test failures on master (Talat and Ted)

[syuanjiang]

HBASE-15939 Two shell test failures on master (Talat and Ted)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72d3f2a8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72d3f2a8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72d3f2a8

Branch: refs/heads/hbase-12439
Commit: 72d3f2a86825808af879ed77a9fafe4c42daed6a
Parents: fc890a2
Author: tedyu 
Authored: Thu Jun 2 15:44:18 2016 -0700
Committer: tedyu 
Committed: Thu Jun 2 15:44:18 2016 -0700

--
 hbase-shell/src/main/ruby/hbase/hbase.rb | 2 +-
 .../java/org/apache/hadoop/hbase/client/AbstractTestShell.java   | 4 
 2 files changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/72d3f2a8/hbase-shell/src/main/ruby/hbase/hbase.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/hbase.rb 
b/hbase-shell/src/main/ruby/hbase/hbase.rb
index 797d784..852f349 100644
--- a/hbase-shell/src/main/ruby/hbase/hbase.rb
+++ b/hbase-shell/src/main/ruby/hbase/hbase.rb
@@ -58,7 +58,7 @@ module Hbase
 
 # Create new one each time
 def table(table, shell)
-  ::Hbase::Table.new(@connection.getTable(table), shell)
+  ::Hbase::Table.new(@connection.getTable(TableName.valueOf(table)), shell)
 end
 
 def replication_admin(formatter)

http://git-wip-us.apache.org/repos/asf/hbase/blob/72d3f2a8/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
--
diff --git 
a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
 
b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
index ddcc1ea..87d14dd 100644
--- 
a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
+++ 
b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
@@ -44,6 +44,10 @@ public abstract class AbstractTestShell {
 
TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, 
false);
 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
 
+// Below settings are necessary for task monitor test.
+TEST_UTIL.getConfiguration().setInt(HConstants.MASTER_INFO_PORT, 0);
+TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_INFO_PORT, 0);
+
TEST_UTIL.getConfiguration().setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, 
true);
 // Security setup configuration
 SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
 VisibilityTestUtil.enableVisiblityLabels(TEST_UTIL.getConfiguration());

[08/50] hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess

[syuanjiang]

HBASE-15931 Add log for long-running tasks in AsyncProcess


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/53eb27bb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/53eb27bb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/53eb27bb

Branch: refs/heads/hbase-12439
Commit: 53eb27bb60bc736f0c56c7a2facfb6e6ccb91be5
Parents: cbb95cd
Author: Yu Li 
Authored: Thu Jun 2 12:00:42 2016 +0800
Committer: Yu Li 
Committed: Thu Jun 2 12:00:42 2016 +0800

--
 .../hadoop/hbase/client/AsyncProcess.java   | 53 
 .../hbase/client/BufferedMutatorImpl.java   |  3 +-
 .../hadoop/hbase/client/TestAsyncProcess.java   |  5 +-
 3 files changed, 49 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/53eb27bb/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index 6f7ba59..812e4bf 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -120,6 +120,12 @@ class AsyncProcess {
*/
   public static final String LOG_DETAILS_FOR_BATCH_ERROR = 
"hbase.client.log.batcherrors.details";
 
+  private final int thresholdToLogUndoneTaskDetails;
+  private static final String THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS =
+  "hbase.client.threshold.log.details";
+  private static final int DEFAULT_THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS = 10;
+  private final int THRESHOLD_TO_LOG_REGION_DETAILS = 2;
+
   /**
* The context used to wait for results from one submit call.
* 1) If AsyncProcess is set to track errors globally, and not per call (for 
HTable puts),
@@ -332,6 +338,10 @@ class AsyncProcess {
 this.rpcCallerFactory = rpcCaller;
 this.rpcFactory = rpcFactory;
 this.logBatchErrorDetails = conf.getBoolean(LOG_DETAILS_FOR_BATCH_ERROR, 
false);
+
+this.thresholdToLogUndoneTaskDetails =
+conf.getInt(THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS,
+  DEFAULT_THRESHOLD_TO_LOG_UNDONE_TASK_DETAILS);
   }
 
   /**
@@ -389,7 +399,7 @@ class AsyncProcess {
 List locationErrorRows = null;
 do {
   // Wait until there is at least one slot for a new task.
-  waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1);
+  waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1, 
tableName.getNameAsString());
 
   // Remember the previous decisions about regions or region servers we 
put in the
   //  final multi.
@@ -1765,18 +1775,19 @@ class AsyncProcess {
   @VisibleForTesting
   /** Waits until all outstanding tasks are done. Used in tests. */
   void waitUntilDone() throws InterruptedIOException {
-waitForMaximumCurrentTasks(0);
+waitForMaximumCurrentTasks(0, null);
   }
 
   /** Wait until the async does not have more than max tasks in progress. */
-  private void waitForMaximumCurrentTasks(int max) throws 
InterruptedIOException {
-waitForMaximumCurrentTasks(max, tasksInProgress, id);
+  private void waitForMaximumCurrentTasks(int max, String tableName)
+  throws InterruptedIOException {
+waitForMaximumCurrentTasks(max, tasksInProgress, id, tableName);
   }
 
   // Break out this method so testable
   @VisibleForTesting
-  static void waitForMaximumCurrentTasks(int max, final AtomicLong 
tasksInProgress, final long id)
-  throws InterruptedIOException {
+  void waitForMaximumCurrentTasks(int max, final AtomicLong tasksInProgress, 
final long id,
+  String tableName) throws InterruptedIOException {
 long lastLog = EnvironmentEdgeManager.currentTime();
 long currentInProgress, oldInProgress = Long.MAX_VALUE;
 while ((currentInProgress = tasksInProgress.get()) > max) {
@@ -1785,7 +1796,11 @@ class AsyncProcess {
 if (now > lastLog + 1) {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
-  + max + ", tasksInProgress=" + currentInProgress);
+  + max + ", tasksInProgress=" + currentInProgress +
+  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
+logDetailsOfUndoneTasks(currentInProgress);
+  }
 }
   }
   oldInProgress = currentInProgress;
@@ -1802,6 +1817,25 @@ class AsyncProcess {
 }
   }
 
+  private void logDetailsOfUndoneTasks(long taskInProgress) {
+ArrayList servers = new ArrayList();
+for (Map.Entry

[01/50] hbase git commit: HBASE-15918 Cleanup excludes/includes file after use in hbase-personality.sh to avoid asf license error. (Apekshit)

[syuanjiang]

Repository: hbase
Updated Branches:
  refs/heads/hbase-12439 a11091c49 -> bd45cf347


HBASE-15918 Cleanup excludes/includes file after use in hbase-personality.sh to 
avoid asf license error. (Apekshit)

Change-Id: I3a47113ef9f598f90357b7cb580cbdc56f3c46f4

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75c23605
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75c23605
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75c23605

Branch: refs/heads/hbase-12439
Commit: 75c23605430266da0f30eef04b97ebd4b30c60b8
Parents: f0a1e22
Author: Apekshit 
Authored: Mon May 30 20:22:29 2016 -0700
Committer: stack 
Committed: Mon May 30 21:30:19 2016 -0700

--
 dev-support/hbase-personality.sh | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/75c23605/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 6797090..3b6ebad 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -104,6 +104,7 @@ function personality_modules
   if [[ -n "${excludes}" ]]; then
 extra="${extra} -Dtest.exclude.pattern=${excludes}"
   fi
+  rm excludes
 else
   echo "Wget error $? in fetching excludes file from url" \
"${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
@@ -115,6 +116,7 @@ function personality_modules
   if [[ -n "${includes}" ]]; then
 extra="${extra} -Dtest=${includes}"
   fi
+  rm includes
 else
   echo "Wget error $? in fetching includes file from url" \
"${INCLUDE_TESTS_URL}. Ignoring and proceeding."

[10/50] hbase git commit: HBASE-15933 NullPointerException may be thrown from SimpleRegionNormalizer#getRegionSize()

[syuanjiang]

HBASE-15933 NullPointerException may be thrown from 
SimpleRegionNormalizer#getRegionSize()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfe868d5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfe868d5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfe868d5

Branch: refs/heads/hbase-12439
Commit: cfe868d56eeb0367c2fcf4a18a1d06c57abb7e54
Parents: a0f49c9
Author: tedyu 
Authored: Thu Jun 2 01:55:28 2016 -0700
Committer: tedyu 
Committed: Thu Jun 2 01:55:28 2016 -0700

--
 .../hbase/master/normalizer/SimpleRegionNormalizer.java  | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cfe868d5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 583f873..d209eb7 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -140,7 +140,9 @@ public class SimpleRegionNormalizer implements 
RegionNormalizer {
 for (int i = 0; i < tableRegions.size(); i++) {
   HRegionInfo hri = tableRegions.get(i);
   long regionSize = getRegionSize(hri);
-  totalSizeMb += regionSize;
+  if (regionSize > 0) {
+totalSizeMb += regionSize;
+  }
 }
 
 double avgRegionSize = totalSizeMb / (double) tableRegions.size();
@@ -204,6 +206,10 @@ public class SimpleRegionNormalizer implements 
RegionNormalizer {
   getRegionServerOfRegion(hri);
 RegionLoad regionLoad = masterServices.getServerManager().getLoad(sn).
   getRegionsLoad().get(hri.getRegionName());
+if (regionLoad == null) {
+  LOG.debug(hri.getRegionNameAsString() + " was not found in RegionsLoad");
+  return -1;
+}
 return regionLoad.getStorefileSizeMB();
   }
 }

hbase git commit: HBASE-16006 FileSystem should be obtained from specified path in WALInputFormat#getSplits()

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 5901cbbe6 -> 403697cd6


HBASE-16006 FileSystem should be obtained from specified path in 
WALInputFormat#getSplits()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/403697cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/403697cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/403697cd

Branch: refs/heads/HBASE-7912
Commit: 403697cd6881f6cc6591d076399a05b2bd285e3e
Parents: 5901cbb
Author: tedyu 
Authored: Fri Jun 10 15:16:48 2016 -0700
Committer: Ted 
Committed: Fri Jun 10 15:16:48 2016 -0700

--
 .../java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java| 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/403697cd/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
index 736b8a5..2fa7a70 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java
@@ -239,10 +239,9 @@ public class WALInputFormat extends InputFormat {
 long startTime = conf.getLong(startKey, Long.MIN_VALUE);
 long endTime = conf.getLong(endKey, Long.MAX_VALUE);
 
-FileSystem fs = FileSystem.get(conf);
-
 List allFiles = new ArrayList();
 for(Path inputPath: inputPaths){
+  FileSystem fs = inputPath.getFileSystem(conf);
   List files = getFiles(fs, inputPath, startTime, endTime);
   allFiles.addAll(files);
 }

hbase git commit: HBASE-16004 Update to Netty 4.1.1

[stack]

Repository: hbase
Updated Branches:
  refs/heads/master 6da6babe4 -> bd45cf347


HBASE-16004 Update to Netty 4.1.1

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd45cf34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd45cf34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd45cf34

Branch: refs/heads/master
Commit: bd45cf34762332a3a51f605798a3e050e7a1e62e
Parents: 6da6bab
Author: Jurriaan Mous 
Authored: Fri Jun 10 17:57:42 2016 +0200
Committer: stack 
Committed: Fri Jun 10 12:14:05 2016 -0700

--
 .../main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java| 4 ++--
 pom.xml  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index c1ed748..723a234 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -347,13 +347,13 @@ public class AsyncRpcClient extends AbstractRpcClient {
 }
 // do not close global EventLoopGroup.
 if (!useGlobalEventLoopGroup) {
-  bootstrap.group().shutdownGracefully();
+  bootstrap.config().group().shutdownGracefully();
 }
   }
 
   @Override
   public EventLoop getEventExecutor() {
-return this.bootstrap.group().next();
+return this.bootstrap.config().group().next();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/bd45cf34/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 0e33ae8..93b7a0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1254,7 +1254,7 @@
 4.0.3
 2.4.1
 1.3.3
-4.0.30.Final
+4.1.1.Final
 3.6.2.Final
 2.1.2
 1.0.8

hbase git commit: HBASE-15946 Eliminate possible security concerns in RS web UI's store file metrics (Sean Mackrory)

[antonov]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 f245fe106 -> d2d3dcdae


HBASE-15946 Eliminate possible security concerns in RS web UI's store file 
metrics (Sean Mackrory)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2d3dcda
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2d3dcda
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2d3dcda

Branch: refs/heads/branch-1.2
Commit: d2d3dcdaec0412614badf77f866b89256296d8f4
Parents: f245fe1
Author: Sean Mackrory 
Authored: Tue May 31 10:28:27 2016 -0600
Committer: Mikhail Antonov 
Committed: Fri Jun 10 10:35:37 2016 -0700

--
 .../hbase/io/hfile/HFilePrettyPrinter.java  | 111 ---
 .../hbase-webapps/regionserver/storeFile.jsp|  35 +++---
 2 files changed, 86 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d3dcda/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index f083f8d..e7cfa7b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -1,4 +1,3 @@
-
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -97,6 +96,9 @@ public class HFilePrettyPrinter extends Configured implements 
Tool {
   private boolean checkFamily;
   private boolean isSeekToRow = false;
 
+  private PrintStream out = System.out;
+  private PrintStream err = System.err;
+
   /**
* The row which the user wants to specify and print all the KeyValues for.
*/
@@ -140,6 +142,11 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 options.addOptionGroup(files);
   }
 
+  public void setPrintStreams(PrintStream out, PrintStream err) {
+this.out = out;
+this.err = err;
+  }
+
   public boolean parseOptions(String args[]) throws ParseException,
   IOException {
 if (args.length == 0) {
@@ -170,7 +177,7 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 row = Bytes.toBytesBinary(key);
 isSeekToRow = true;
   } else {
-System.err.println("Invalid row is specified.");
+err.println("Invalid row is specified.");
 System.exit(-1);
   }
 }
@@ -184,17 +191,17 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
   String enc = HRegionInfo.encodeRegionName(rn);
   Path regionDir = new Path(tableDir, enc);
   if (verbose)
-System.out.println("region dir -> " + regionDir);
+out.println("region dir -> " + regionDir);
   List regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()),
   regionDir);
   if (verbose)
-System.out.println("Number of region files found -> "
+out.println("Number of region files found -> "
 + regionFiles.size());
   if (verbose) {
 int i = 1;
 for (Path p : regionFiles) {
   if (verbose)
-System.out.println("Found file[" + i++ + "] -> " + p);
+out.println("Found file[" + i++ + "] -> " + p);
 }
   }
   files.addAll(regionFiles);
@@ -227,27 +234,46 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 // iterate over all files found
 for (Path fileName : files) {
   try {
-processFile(fileName);
+int exitCode = processFile(fileName);
+if (exitCode != 0) {
+  return exitCode;
+}
   } catch (IOException ex) {
 LOG.error("Error reading " + fileName, ex);
-System.exit(-2);
+return -2;
   }
 }
 
 if (verbose || printKey) {
-  System.out.println("Scanned kv count -> " + count);
+  out.println("Scanned kv count -> " + count);
 }
 
 return 0;
   }
 
-  private void processFile(Path file) throws IOException {
+  public int processFile(Path file) throws IOException {
 if (verbose)
-  System.out.println("Scanning -> " + file);
+  out.println("Scanning -> " + file);
+
+Path rootPath = FSUtils.getRootDir(getConf());
+String rootString = rootPath + rootPath.SEPARATOR;
+if (!file.toString().startsWith(rootString)) {
+  // First we see if fully-qualified URI matches the root dir. It might
+  // also be an absolute path in the same filesystem, so we prepend the FS
+  // of the root dir and see if that fully-qualified URI matches.
+  FileSystem rootFS =