hbase git commit: HBASE-17606 Fix failing TestRpcControllerFactory introduced by HBASE-17508

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/master 9a78d0088 -> 5c77a7dcd


HBASE-17606 Fix failing TestRpcControllerFactory introduced by HBASE-17508


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5c77a7dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5c77a7dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5c77a7dc

Branch: refs/heads/master
Commit: 5c77a7dcd455f7a6e0ba3f289266032be687dc4f
Parents: 9a78d00
Author: zhangduo 
Authored: Tue Feb 7 11:16:07 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 14:31:45 2017 +0800

--
 .../org/apache/hadoop/hbase/client/TestRpcControllerFactory.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5c77a7dc/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java
--
diff --git 
a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java
 
b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java
index 3fdd8cb..14a2bf9 100644
--- 
a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java
+++ 
b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/client/TestRpcControllerFactory.java
@@ -170,7 +170,7 @@ public class TestRpcControllerFactory {
 ResultScanner scan = table.getScanner(fam1);
 scan.next();
 scan.close();
-counter = verifyCount(counter + 2);
+counter = verifyCount(counter + 1);
 
 Get g2 = new Get(row);
 table.get(Lists.newArrayList(g, g2));
@@ -189,7 +189,7 @@ public class TestRpcControllerFactory {
 
 // reversed, regular
 scanInfo.setSmall(false);
-counter = doScan(table, scanInfo, counter + 2);
+counter = doScan(table, scanInfo, counter + 1);
 
 table.close();
 connection.close();

hbase git commit: HBASE-17264 Processing RIT with offline state will always fail to open the first time (Allan Yang)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 637fbb496 -> 4348dd602


HBASE-17264 Processing RIT with offline state will always fail to open the 
first time (Allan Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4348dd60
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4348dd60
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4348dd60

Branch: refs/heads/branch-1.1
Commit: 4348dd60202f1ec9bc319f71436debac0637ce48
Parents: 637fbb4
Author: tedyu 
Authored: Mon Feb 6 20:28:16 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 20:28:16 2017 -0800

--
 .../hadoop/hbase/master/AssignmentManager.java  |  7 ++---
 .../hbase/master/TestAssignmentManager.java | 27 
 2 files changed, 31 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4348dd60/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 0cbc005..18babe6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -847,7 +847,7 @@ public class AssignmentManager extends ZooKeeperListener {
 
   case M_ZK_REGION_OFFLINE:
 // Insert in RIT and resend to the regionserver
-regionStates.updateRegionState(rt, State.PENDING_OPEN);
+regionStates.updateRegionState(rt, State.OFFLINE);
 final RegionState rsOffline = regionStates.getRegionState(regionInfo);
 this.executorService.submit(
   new EventHandler(server, EventType.M_MASTER_RECOVERY) {
@@ -857,7 +857,7 @@ public class AssignmentManager extends ZooKeeperListener {
   try {
 RegionPlan plan = new RegionPlan(regionInfo, null, sn);
 addPlan(encodedName, plan);
-assign(rsOffline, false, false);
+assign(rsOffline, true, false);
   } finally {
 lock.unlock();
   }
@@ -1576,6 +1576,7 @@ public class AssignmentManager extends ZooKeeperListener {
   /**
* Use care with forceNewPlan. It could cause double assignment.
*/
+  @VisibleForTesting
   public void assign(HRegionInfo region,
   boolean setOfflineInZK, boolean forceNewPlan) {
 if (isDisabledorDisablingRegionInRIT(region)) {
@@ -2053,7 +2054,7 @@ public class AssignmentManager extends ZooKeeperListener {
* @param setOfflineInZK
* @param forceNewPlan
*/
-  private void assign(RegionState state,
+  public void assign(RegionState state,
   boolean setOfflineInZK, final boolean forceNewPlan) {
 long startTime = EnvironmentEdgeManager.currentTime();
 try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/4348dd60/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
index 0dd1ff2..d585756 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
@@ -859,6 +859,7 @@ public class TestAssignmentManager {
 }
   }
 
+
   /**
* Mocked load balancer class used in the testcase to make sure that the 
testcase waits until
* random assignment is called and the gate variable is set to true.
@@ -929,6 +930,22 @@ public class TestAssignmentManager {
 am.shutdown();
   }
 
+  @Test(timeout = 60)
+  public void testAssignmentOfRegionRITWithOffline() throws IOException,
+  KeeperException, ServiceException, CoordinatedStateException, 
InterruptedException {
+AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
+this.server, this.serverManager);
+ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_B);
+am.gate.set(false);
+// join the cluster - that's when the AM is really kicking in after a 
restart
+am.joinCluster();
+while (!am.gate.get()) {
+  Thread.sleep(10);
+}
+assertTrue(am.getRegionStates().getRegionState(REGIONINFO).getState()
+== RegionState.State.PENDING_OPEN);
+am.shutdown();
+  }
   /**
* Test the scenario when the master is in failover and trying to process a
* region which is in Opening state 

hbase git commit: HBASE-17264 Processing RIT with offline state will always fail to open the first time (Allan Yang)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 b1690e176 -> 27303fdfb


HBASE-17264 Processing RIT with offline state will always fail to open the 
first time (Allan Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27303fdf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27303fdf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27303fdf

Branch: refs/heads/branch-1.2
Commit: 27303fdfb7180d8ba8d8241dc7217a35cc310994
Parents: b1690e1
Author: tedyu 
Authored: Mon Feb 6 20:25:05 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 20:25:05 2017 -0800

--
 .../org/apache/hadoop/hbase/master/AssignmentManager.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/27303fdf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 29eacf0..d1ac1bb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -847,7 +847,7 @@ public class AssignmentManager extends ZooKeeperListener {
 
   case M_ZK_REGION_OFFLINE:
 // Insert in RIT and resend to the regionserver
-regionStates.updateRegionState(rt, State.PENDING_OPEN);
+regionStates.updateRegionState(rt, State.OFFLINE);
 final RegionState rsOffline = regionStates.getRegionState(regionInfo);
 this.executorService.submit(
   new EventHandler(server, EventType.M_MASTER_RECOVERY) {
@@ -857,7 +857,7 @@ public class AssignmentManager extends ZooKeeperListener {
   try {
 RegionPlan plan = new RegionPlan(regionInfo, null, sn);
 addPlan(encodedName, plan);
-assign(rsOffline, false, false);
+assign(rsOffline, true, false);
   } finally {
 lock.unlock();
   }
@@ -1581,6 +1581,7 @@ public class AssignmentManager extends ZooKeeperListener {
   /**
* Use care with forceNewPlan. It could cause double assignment.
*/
+  @VisibleForTesting
   public void assign(HRegionInfo region,
   boolean setOfflineInZK, boolean forceNewPlan) {
 if (isDisabledorDisablingRegionInRIT(region)) {
@@ -2058,7 +2059,7 @@ public class AssignmentManager extends ZooKeeperListener {
* @param setOfflineInZK
* @param forceNewPlan
*/
-  private void assign(RegionState state,
+  public void assign(RegionState state,
   boolean setOfflineInZK, final boolean forceNewPlan) {
 long startTime = EnvironmentEdgeManager.currentTime();
 try {

hbase git commit: HBASE-17264 Processing RIT with offline state will always fail to open the first time (Allan Yang)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 41dc37428 -> daa4940ef


HBASE-17264 Processing RIT with offline state will always fail to open the 
first time (Allan Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/daa4940e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/daa4940e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/daa4940e

Branch: refs/heads/branch-1.3
Commit: daa4940ef1f5b1153629370d9aea07ad6e3d1794
Parents: 41dc374
Author: tedyu 
Authored: Mon Feb 6 19:49:08 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 19:49:08 2017 -0800

--
 .../org/apache/hadoop/hbase/master/AssignmentManager.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/daa4940e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 1af826e..5c040dd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -848,7 +848,7 @@ public class AssignmentManager extends ZooKeeperListener {
 
   case M_ZK_REGION_OFFLINE:
 // Insert in RIT and resend to the regionserver
-regionStates.updateRegionState(rt, State.PENDING_OPEN);
+regionStates.updateRegionState(rt, State.OFFLINE);
 final RegionState rsOffline = regionStates.getRegionState(regionInfo);
 this.executorService.submit(
   new EventHandler(server, EventType.M_MASTER_RECOVERY) {
@@ -858,7 +858,7 @@ public class AssignmentManager extends ZooKeeperListener {
   try {
 RegionPlan plan = new RegionPlan(regionInfo, null, sn);
 addPlan(encodedName, plan);
-assign(rsOffline, false, false);
+assign(rsOffline, true, false);
   } finally {
 lock.unlock();
   }
@@ -1582,6 +1582,7 @@ public class AssignmentManager extends ZooKeeperListener {
   /**
* Use care with forceNewPlan. It could cause double assignment.
*/
+  @VisibleForTesting
   public void assign(HRegionInfo region,
   boolean setOfflineInZK, boolean forceNewPlan) {
 if (isDisabledorDisablingRegionInRIT(region)) {
@@ -2059,7 +2060,7 @@ public class AssignmentManager extends ZooKeeperListener {
* @param setOfflineInZK
* @param forceNewPlan
*/
-  private void assign(RegionState state,
+  public void assign(RegionState state,
   boolean setOfflineInZK, final boolean forceNewPlan) {
 long startTime = EnvironmentEdgeManager.currentTime();
 try {

hbase git commit: HBASE-17264 Processing RIT with offline state will always fail to open the first time (Allan Yang)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 f708d986a -> 0a0aef345


HBASE-17264 Processing RIT with offline state will always fail to open the 
first time (Allan Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a0aef34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a0aef34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a0aef34

Branch: refs/heads/branch-1
Commit: 0a0aef345a4b4f4deaa66434f749a3abb248846b
Parents: f708d98
Author: tedyu 
Authored: Mon Feb 6 19:48:14 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 19:48:14 2017 -0800

--
 .../org/apache/hadoop/hbase/master/AssignmentManager.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0a0aef34/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2035f6a..f3c0787 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -874,7 +874,7 @@ public class AssignmentManager extends ZooKeeperListener {
 
   case M_ZK_REGION_OFFLINE:
 // Insert in RIT and resend to the regionserver
-regionStates.updateRegionState(rt, State.PENDING_OPEN);
+regionStates.updateRegionState(rt, State.OFFLINE);
 final RegionState rsOffline = regionStates.getRegionState(regionInfo);
 this.executorService.submit(
   new EventHandler(server, EventType.M_MASTER_RECOVERY) {
@@ -884,7 +884,7 @@ public class AssignmentManager extends ZooKeeperListener {
   try {
 RegionPlan plan = new RegionPlan(regionInfo, null, sn);
 addPlan(encodedName, plan);
-assign(rsOffline, false, false);
+assign(rsOffline, true, false);
   } finally {
 lock.unlock();
   }
@@ -1608,6 +1608,7 @@ public class AssignmentManager extends ZooKeeperListener {
   /**
* Use care with forceNewPlan. It could cause double assignment.
*/
+  @VisibleForTesting
   public void assign(HRegionInfo region,
   boolean setOfflineInZK, boolean forceNewPlan) {
 if (isDisabledorDisablingRegionInRIT(region)) {
@@ -2085,7 +2086,7 @@ public class AssignmentManager extends ZooKeeperListener {
* @param setOfflineInZK
* @param forceNewPlan
*/
-  private void assign(RegionState state,
+  public void assign(RegionState state,
   boolean setOfflineInZK, final boolean forceNewPlan) {
 long startTime = EnvironmentEdgeManager.currentTime();
 try {

hbase git commit: HBASE-17389 Convert all internal usages from ReplicationAdmin to Admin

[zghao]

Repository: hbase
Updated Branches:
  refs/heads/master af9d359b8 -> 9a78d0088


HBASE-17389 Convert all internal usages from ReplicationAdmin to Admin


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a78d008
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a78d008
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a78d008

Branch: refs/heads/master
Commit: 9a78d008841726ec2029215cddf0c0b2141771ae
Parents: af9d359
Author: Guanghao Zhang 
Authored: Tue Feb 7 10:18:59 2017 +0800
Committer: Guanghao Zhang 
Committed: Tue Feb 7 10:18:59 2017 +0800

--
 .../org/apache/hadoop/hbase/client/Admin.java   | 26 ++
 .../apache/hadoop/hbase/client/HBaseAdmin.java  | 83 +++
 .../client/replication/ReplicationAdmin.java| 33 
 .../hbase/client/replication/TableCFs.java  | 12 +++
 .../org/apache/hadoop/hbase/master/HMaster.java |  2 +
 .../master/cleaner/ReplicationMetaCleaner.java  | 18 ++--
 .../regionserver/DumpReplicationQueues.java | 57 ++---
 .../hbase/util/ServerRegionReplicaUtil.java | 12 +--
 .../replication/TestMasterReplication.java  | 47 ---
 ...sibilityLabelReplicationWithExpAsString.java |  4 +-
 .../TestVisibilityLabelsReplication.java|  6 +-
 .../src/main/ruby/hbase/replication_admin.rb| 63 +++---
 .../src/main/ruby/shell/commands/list_peers.rb  | 16 ++--
 .../shell/commands/list_replicated_tables.rb| 17 ++--
 .../test/ruby/hbase/replication_admin_test.rb   | 86 
 15 files changed, 305 insertions(+), 177 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a78d008/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 232dbf4..cc14acd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.client;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Future;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hbase.quotas.QuotaFilter;
 import org.apache.hadoop.hbase.quotas.QuotaRetriever;
 import org.apache.hadoop.hbase.quotas.QuotaSettings;
 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
+import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
@@ -1907,6 +1909,30 @@ public interface Admin extends Abortable, Closeable {
   }
 
   /**
+   * Append the replicable table-cf config of the specified peer
+   * @param id a short that identifies the cluster
+   * @param tableCfs A map from tableName to column family names
+   * @throws ReplicationException
+   * @throws IOException
+   */
+  default void appendReplicationPeerTableCFs(String id,
+  Map tableCfs) throws 
ReplicationException,
+  IOException {
+  }
+
+  /**
+   * Remove some table-cfs from config of the specified peer
+   * @param id a short name that identifies the cluster
+   * @param tableCfs A map from tableName to column family names
+   * @throws ReplicationException
+   * @throws IOException
+   */
+  default void removeReplicationPeerTableCFs(String id,
+  Map tableCfs) throws 
ReplicationException,
+  IOException {
+  }
+
+  /**
* Return a list of replication peers.
* @return a list of replication peers description
* @throws IOException

http://git-wip-us.apache.org/repos/asf/hbase/blob/9a78d008/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 4e0a6c7..65070b9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -23,10 +23,13 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import 

hbase git commit: HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 62f7b3952 -> 637fbb496


HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/637fbb49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/637fbb49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/637fbb49

Branch: refs/heads/branch-1.1
Commit: 637fbb496c230496d1cf8ba91e59864f4b973d7f
Parents: 62f7b39
Author: zhangduo 
Authored: Tue Feb 7 10:07:05 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 10:07:05 2017 +0800

--
 .../hbase/client/TestMetaWithReplicas.java  | 85 +---
 1 file changed, 56 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/637fbb49/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index 477d10a..433a6eb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hbase.client;
 
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -40,13 +42,14 @@ import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import 
org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.HBaseFsck;
-import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -412,15 +415,16 @@ public class TestMetaWithReplicas {
 assertErrors(hbck, new ERROR_CODE[]{});
   }
 
-  @Test (timeout=18)
+  @Test
   public void testMetaTableReplicaAssignment() throws Exception {
-ClusterConnection c = 
ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
-RegionLocations rl =
+final ClusterConnection c =
+ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
+final RegionLocations rl =
 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
-ServerName meta0SN = rl.getRegionLocation(0).getServerName();
+final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
 LOG.debug("The hbase:meta default replica region is in server: " + 
meta0SN);
-ServerName meta1SN = rl.getRegionLocation(1).getServerName();
+final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
 LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
   " is in server: " + meta1SN);
 
@@ -436,19 +440,30 @@ public class TestMetaWithReplicas {
 LOG.debug("Restarting the master server " + masterSN);
 TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), 
masterSN.getPort());
 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
-rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
 // wait for replica 1 to be re-assigned
-ServerName newMeta1SN;
-int i = 0;
-do {
-  Thread.sleep(100);
-  newMeta1SN = rl.getRegionLocation(1).getServerName();
-  i++;
-} while (meta1SN.equals(newMeta1SN) & i < 600); // wait for 60 seconds
-LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
-" is now moved from server " + meta1SN + " to server " + newMeta1SN);
-assert (!meta1SN.equals(newMeta1SN));
+TEST_UTIL.waitFor(6, 100, new ExplainingPredicate() {
+
+  @Override
+  public boolean 

hbase git commit: HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 6d1082f62 -> b1690e176


HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b1690e17
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b1690e17
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b1690e17

Branch: refs/heads/branch-1.2
Commit: b1690e17614c198498dd06f38bee1f0c4e53dae7
Parents: 6d1082f
Author: zhangduo 
Authored: Sun Feb 5 09:12:50 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 10:01:46 2017 +0800

--
 .../hbase/client/TestMetaWithReplicas.java  | 83 +---
 1 file changed, 55 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b1690e17/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index c38122a..9038783 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -22,6 +22,8 @@ import static 
org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -41,13 +43,14 @@ import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import 
org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.HBaseFsck;
-import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -424,15 +427,16 @@ public class TestMetaWithReplicas {
 assertErrors(hbck, new ERROR_CODE[]{});
   }
 
-  @Test (timeout=18)
+  @Test
   public void testMetaTableReplicaAssignment() throws Exception {
-ClusterConnection c = 
ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
-RegionLocations rl =
+final ClusterConnection c =
+ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
+final RegionLocations rl =
 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
-ServerName meta0SN = rl.getRegionLocation(0).getServerName();
+final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
 LOG.debug("The hbase:meta default replica region is in server: " + 
meta0SN);
-ServerName meta1SN = rl.getRegionLocation(1).getServerName();
+final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
 LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
   " is in server: " + meta1SN);
 
@@ -448,19 +452,30 @@ public class TestMetaWithReplicas {
 LOG.debug("Restarting the master server " + masterSN);
 TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), 
masterSN.getPort());
 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
-rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
 // wait for replica 1 to be re-assigned
-ServerName newMeta1SN;
-int i = 0;
-do {
-  Thread.sleep(100);
-  newMeta1SN = rl.getRegionLocation(1).getServerName();
-  i++;
-} while (meta1SN.equals(newMeta1SN) & i < 600); // wait for 60 seconds
-LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
-" is now moved from server " + meta1SN + " to server " + newMeta1SN);
-assert (!meta1SN.equals(newMeta1SN));
+TEST_UTIL.waitFor(6, 100, new ExplainingPredicate() {
+
+  @Override
+  public boolean evaluate() throws IOException {
+RegionLocations rls =
+

hbase git commit: HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 30b229f24 -> 41dc37428


HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41dc3742
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41dc3742
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41dc3742

Branch: refs/heads/branch-1.3
Commit: 41dc37428b78b93d0830c65bc612d3fe090331d2
Parents: 30b229f
Author: zhangduo 
Authored: Sun Feb 5 09:12:50 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 09:55:08 2017 +0800

--
 .../hbase/client/TestMetaWithReplicas.java  | 83 +---
 1 file changed, 55 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/41dc3742/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index c38122a..9038783 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -22,6 +22,8 @@ import static 
org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -41,13 +43,14 @@ import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import 
org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.HBaseFsck;
-import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -424,15 +427,16 @@ public class TestMetaWithReplicas {
 assertErrors(hbck, new ERROR_CODE[]{});
   }
 
-  @Test (timeout=18)
+  @Test
   public void testMetaTableReplicaAssignment() throws Exception {
-ClusterConnection c = 
ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
-RegionLocations rl =
+final ClusterConnection c =
+ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
+final RegionLocations rl =
 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
-ServerName meta0SN = rl.getRegionLocation(0).getServerName();
+final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
 LOG.debug("The hbase:meta default replica region is in server: " + 
meta0SN);
-ServerName meta1SN = rl.getRegionLocation(1).getServerName();
+final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
 LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
   " is in server: " + meta1SN);
 
@@ -448,19 +452,30 @@ public class TestMetaWithReplicas {
 LOG.debug("Restarting the master server " + masterSN);
 TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), 
masterSN.getPort());
 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
-rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
 // wait for replica 1 to be re-assigned
-ServerName newMeta1SN;
-int i = 0;
-do {
-  Thread.sleep(100);
-  newMeta1SN = rl.getRegionLocation(1).getServerName();
-  i++;
-} while (meta1SN.equals(newMeta1SN) & i < 600); // wait for 60 seconds
-LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
-" is now moved from server " + meta1SN + " to server " + newMeta1SN);
-assert (!meta1SN.equals(newMeta1SN));
+TEST_UTIL.waitFor(6, 100, new ExplainingPredicate() {
+
+  @Override
+  public boolean evaluate() throws IOException {
+RegionLocations rls =
+

hbase git commit: HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 7cac85b66 -> f708d986a


HBASE-17597 TestMetaWithReplicas.testMetaTableReplicaAssignment is flaky


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f708d986
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f708d986
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f708d986

Branch: refs/heads/branch-1
Commit: f708d986a035f0fbbd8d399187f7cd100b479548
Parents: 7cac85b
Author: zhangduo 
Authored: Sun Feb 5 09:12:50 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 09:51:46 2017 +0800

--
 .../hbase/client/TestMetaWithReplicas.java  | 83 +---
 1 file changed, 55 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f708d986/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index bf81301..f466093 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -22,6 +22,8 @@ import static 
org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -41,13 +43,14 @@ import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import 
org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation;
 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.HBaseFsck;
-import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.HBaseFsckRepair;
 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -424,15 +427,16 @@ public class TestMetaWithReplicas {
 assertErrors(hbck, new ERROR_CODE[]{});
   }
 
-  @Test (timeout=18)
+  @Test
   public void testMetaTableReplicaAssignment() throws Exception {
-ClusterConnection c = 
ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
-RegionLocations rl =
+final ClusterConnection c =
+ConnectionManager.getConnectionInternal(TEST_UTIL.getConfiguration());
+final RegionLocations rl =
 c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
-ServerName meta0SN = rl.getRegionLocation(0).getServerName();
+final ServerName meta0SN = rl.getRegionLocation(0).getServerName();
 LOG.debug("The hbase:meta default replica region is in server: " + 
meta0SN);
-ServerName meta1SN = rl.getRegionLocation(1).getServerName();
+final ServerName meta1SN = rl.getRegionLocation(1).getServerName();
 LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
   " is in server: " + meta1SN);
 
@@ -448,19 +452,30 @@ public class TestMetaWithReplicas {
 LOG.debug("Restarting the master server " + masterSN);
 TEST_UTIL.getHBaseClusterInterface().startMaster(masterSN.getHostname(), 
masterSN.getPort());
 TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
-rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, 
false, true);
 
 // wait for replica 1 to be re-assigned
-ServerName newMeta1SN;
-int i = 0;
-do {
-  Thread.sleep(100);
-  newMeta1SN = rl.getRegionLocation(1).getServerName();
-  i++;
-} while (meta1SN.equals(newMeta1SN) & i < 600); // wait for 60 seconds
-LOG.debug("The hbase:meta replica 1 region " + 
rl.getRegionLocation(1).getRegionInfo() +
-" is now moved from server " + meta1SN + " to server " + newMeta1SN);
-assert (!meta1SN.equals(newMeta1SN));
+TEST_UTIL.waitFor(6, 100, new ExplainingPredicate() {
+
+  @Override
+  public boolean evaluate() throws IOException {
+RegionLocations rls =
+

hbase git commit: HBASE-17402 TestAsyncTableScan sometimes hangs

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/master 9ec0ec492 -> af9d359b8


HBASE-17402 TestAsyncTableScan sometimes hangs


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af9d359b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af9d359b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af9d359b

Branch: refs/heads/master
Commit: af9d359b8eb7933bd20f1dcaa2ddb1b27059d23e
Parents: 9ec0ec4
Author: zhangduo 
Authored: Tue Feb 7 09:31:38 2017 +0800
Committer: zhangduo 
Committed: Tue Feb 7 09:42:42 2017 +0800

--
 .../hbase/client/AsyncNonMetaRegionLocator.java | 56 ++--
 .../client/TestAsyncNonMetaRegionLocator.java   | 36 +++--
 2 files changed, 60 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/af9d359b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java
index 27e8cc4..dcf2c91 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Scan.ReadType;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -140,8 +139,8 @@ class AsyncNonMetaRegionLocator {
   return;
 }
 tableCache.cache.computeIfPresent(loc.getRegionInfo().getStartKey(), (k, 
oldLoc) -> {
-  if (oldLoc.getSeqNum() > loc.getSeqNum()
-  || !oldLoc.getServerName().equals(loc.getServerName())) {
+  if (oldLoc.getSeqNum() > loc.getSeqNum() ||
+  !oldLoc.getServerName().equals(loc.getServerName())) {
 return oldLoc;
   }
   return null;
@@ -158,11 +157,11 @@ class AsyncNonMetaRegionLocator {
 if (oldLoc == null) {
   return true;
 }
-if (oldLoc.getSeqNum() > loc.getSeqNum()
-|| oldLoc.getServerName().equals(loc.getServerName())) {
+if (oldLoc.getSeqNum() > loc.getSeqNum() ||
+oldLoc.getServerName().equals(loc.getServerName())) {
   if (LOG.isTraceEnabled()) {
-LOG.trace("Will not add " + loc + " to cache because the old value " + 
oldLoc
-+ " is newer than us or has the same server name");
+LOG.trace("Will not add " + loc + " to cache because the old value " + 
oldLoc +
+" is newer than us or has the same server name");
   }
   return false;
 }
@@ -171,9 +170,9 @@ class AsyncNonMetaRegionLocator {
 return loc;
   }
   if (LOG.isTraceEnabled()) {
-LOG.trace("Will not add " + loc + " to cache because the old value " + 
oldValue
-+ " is newer than us or has the same server name."
-+ " Maybe it is updated before we replace it");
+LOG.trace("Will not add " + loc + " to cache because the old value " + 
oldValue +
+" is newer than us or has the same server name." +
+" Maybe it is updated before we replace it");
   }
   return oldValue;
 });
@@ -217,8 +216,8 @@ class AsyncNonMetaRegionLocator {
   Throwable error) {
 if (error != null) {
   if (LOG.isDebugEnabled()) {
-LOG.debug("Failed to locate region in '" + tableName + "', row='"
-+ Bytes.toStringBinary(req.row) + "', locateType=" + 
req.locateType,
+LOG.debug("Failed to locate region in '" + tableName + "', row='" +
+Bytes.toStringBinary(req.row) + "', locateType=" + req.locateType,
   error);
   }
 }
@@ -250,14 +249,15 @@ class AsyncNonMetaRegionLocator {
   }
 }
   }
-  if (!tableCache.allRequests.isEmpty()
-  && tableCache.hasQuota(maxConcurrentLocateRequestPerTable)) {
+  if (!tableCache.allRequests.isEmpty() &&
+  tableCache.hasQuota(maxConcurrentLocateRequestPerTable)) {
 LocateRequest[] candidates = tableCache.allRequests.keySet().stream()
 .filter(r -> 
!tableCache.isPending(r)).toArray(LocateRequest[]::new);
 if (candidates.length > 0) {
   // TODO: use a better algorithm to send a request which is more 
likely to fetch a new
   // location.
   toSend = 
candidates[ThreadLocalRandom.current().nextInt(candidates.length)];
+  tableCache.send(toSend);
 }
   }
 

[2/2] hbase git commit: HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the application

[enis]

HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the 
application


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/62f7b395
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/62f7b395
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/62f7b395

Branch: refs/heads/branch-1.1
Commit: 62f7b395279301999b374392a2d9824ae191485f
Parents: 4307886
Author: Enis Soztutar 
Authored: Mon Feb 6 12:00:05 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 16:14:21 2017 -0800

--
 .../hadoop/hbase/client/ClientScanner.java  | 20 +-
 .../hbase/regionserver/RSRpcServices.java   | 12 +++-
 .../hadoop/hbase/client/TestFromClientSide.java | 76 +++-
 3 files changed, 103 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/62f7b395/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 4a5e635..2c06fd5 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -401,6 +401,9 @@ public class ClientScanner extends AbstractClientScanner {
 // it doesn't tell us otherwise. We rely on the size or count of results
 boolean serverHasMoreResults = false;
 boolean allResultsSkipped = false;
+// Even if we are retrying due to UnknownScannerException, 
ScannerResetException, etc. we should
+// make sure that we are not retrying indefinitely.
+int retriesLeft = getRetries();
 do {
   allResultsSkipped = false;
   try {
@@ -425,8 +428,18 @@ public class ClientScanner extends AbstractClientScanner {
 // An exception was thrown which makes any partial results that we 
were collecting
 // invalid. The scanner will need to be reset to the beginning of a 
row.
 clearPartialResults();
-// DNRIOEs are thrown to make us break out of retries. Some types of 
DNRIOEs want us
-// to reset the scanner and come back in again.
+
+// Unfortunately, DNRIOE is used in two different semantics.
+// (1) The first is to close the client scanner and bubble up the 
exception all the way
+// to the application. This is preferred when the exception is really 
un-recoverable
+// (like CorruptHFileException, etc). Plain DoNotRetryIOException also 
falls into this
+// bucket usually.
+// (2) Second semantics is to close the current region scanner only, 
but continue the
+// client scanner by overriding the exception. This is usually 
UnknownScannerException,
+// OutOfOrderScannerNextException, etc where the region scanner has to 
be closed, but the
+// application-level ClientScanner has to continue without bubbling up 
the exception to
+// the client. See RSRpcServices to see how it throws DNRIOE's.
+// See also: HBASE-16604, HBASE-17187
 
 // If exception is any but the list below throw it back to the client; 
else setup
 // the scanner and retry.
@@ -438,6 +451,9 @@ public class ClientScanner extends AbstractClientScanner {
 e instanceof ScannerResetException) {
   // Pass. It is easier writing the if loop test as list of what is 
allowed rather than
   // as a list of what is not allowed... so if in here, it means we do 
not throw.
+  if (retriesLeft-- <= 0) {
+throw e; // no more retries
+  }
 } else {
   throw e;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/62f7b395/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index fed9e62..6845a92 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -2540,7 +2540,17 @@ public class RSRpcServices implements 
HBaseRPCErrorHandler,
   // row that the client has last seen.
   closeScanner(region, scanner, scannerName);
 
-  // rethrow DoNotRetryIOException. This can avoid the retry in 
ClientScanner.
+  // If it is a DoNotRetryIOException already, throw as it is. 
Unfortunately, DNRIOE is
+  

[1/2] hbase git commit: HBASE-17587 Do not Rethrow DoNotRetryIOException as UnknownScannerException

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 84d7318f8 -> 62f7b3952


HBASE-17587 Do not Rethrow DoNotRetryIOException as UnknownScannerException

Signed-off-by: Andrew Purtell 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/43078862
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/43078862
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/43078862

Branch: refs/heads/branch-1.1
Commit: 4307886247a206bc0b1d9a12333a3afac3778c1e
Parents: 84d7318
Author: Zach York 
Authored: Thu Feb 2 02:44:58 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 16:14:15 2017 -0800

--
 .../apache/hadoop/hbase/regionserver/RSRpcServices.java | 12 
 1 file changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/43078862/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 116d70c..fed9e62 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.InetSocketAddress;
@@ -2539,6 +2540,17 @@ public class RSRpcServices implements 
HBaseRPCErrorHandler,
   // row that the client has last seen.
   closeScanner(region, scanner, scannerName);
 
+  // rethrow DoNotRetryIOException. This can avoid the retry in 
ClientScanner.
+  if (e instanceof DoNotRetryIOException) {
+throw e;
+  }
+
+  // If it is a FileNotFoundException, wrap as a
+  // DoNotRetryIOException. This can avoid the retry in ClientScanner.
+  if (e instanceof FileNotFoundException) {
+throw new DoNotRetryIOException(e);
+  }
+
   // We closed the scanner already. Instead of throwing the 
IOException, and client
   // retrying with the same scannerId only to get USE on the next RPC, 
we directly throw
   // a special exception to save an RPC.

[1/2] hbase git commit: HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the application

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 9bd76426f -> 6d1082f62


HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the 
application


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d1082f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d1082f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d1082f6

Branch: refs/heads/branch-1.2
Commit: 6d1082f626d7bdc3eb246cb0b9976c84501b16cb
Parents: a07b968
Author: Enis Soztutar 
Authored: Mon Feb 6 12:00:05 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 16:11:53 2017 -0800

--
 .../hadoop/hbase/client/ClientScanner.java  | 20 +-
 .../hbase/regionserver/RSRpcServices.java   | 12 +++-
 .../hadoop/hbase/client/TestFromClientSide.java | 76 +++-
 3 files changed, 103 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1082f6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 944f44e..053814c 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -401,6 +401,9 @@ public class ClientScanner extends AbstractClientScanner {
 // it doesn't tell us otherwise. We rely on the size or count of results
 boolean serverHasMoreResults = false;
 boolean allResultsSkipped = false;
+// Even if we are retrying due to UnknownScannerException, 
ScannerResetException, etc. we should
+// make sure that we are not retrying indefinitely.
+int retriesLeft = getRetries();
 do {
   allResultsSkipped = false;
   try {
@@ -425,8 +428,18 @@ public class ClientScanner extends AbstractClientScanner {
 // An exception was thrown which makes any partial results that we 
were collecting
 // invalid. The scanner will need to be reset to the beginning of a 
row.
 clearPartialResults();
-// DNRIOEs are thrown to make us break out of retries. Some types of 
DNRIOEs want us
-// to reset the scanner and come back in again.
+
+// Unfortunately, DNRIOE is used in two different semantics.
+// (1) The first is to close the client scanner and bubble up the 
exception all the way
+// to the application. This is preferred when the exception is really 
un-recoverable
+// (like CorruptHFileException, etc). Plain DoNotRetryIOException also 
falls into this
+// bucket usually.
+// (2) Second semantics is to close the current region scanner only, 
but continue the
+// client scanner by overriding the exception. This is usually 
UnknownScannerException,
+// OutOfOrderScannerNextException, etc where the region scanner has to 
be closed, but the
+// application-level ClientScanner has to continue without bubbling up 
the exception to
+// the client. See RSRpcServices to see how it throws DNRIOE's.
+// See also: HBASE-16604, HBASE-17187
 
 // If exception is any but the list below throw it back to the client; 
else setup
 // the scanner and retry.
@@ -438,6 +451,9 @@ public class ClientScanner extends AbstractClientScanner {
 e instanceof ScannerResetException) {
   // Pass. It is easier writing the if loop test as list of what is 
allowed rather than
   // as a list of what is not allowed... so if in here, it means we do 
not throw.
+  if (retriesLeft-- <= 0) {
+throw e; // no more retries
+  }
 } else {
   throw e;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1082f6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 46a8120..1d454a1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -2663,7 +2663,17 @@ public class RSRpcServices implements 
HBaseRPCErrorHandler,
   // row that the client has last seen.
   closeScanner(region, scanner, scannerName);
 
-  // rethrow DoNotRetryIOException. This can avoid the retry in 
ClientScanner.
+  // If it is a 

[2/2] hbase git commit: HBASE-17587 Do not Rethrow DoNotRetryIOException as UnknownScannerException

[enis]

HBASE-17587 Do not Rethrow DoNotRetryIOException as UnknownScannerException

Signed-off-by: Andrew Purtell 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a07b9687
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a07b9687
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a07b9687

Branch: refs/heads/branch-1.2
Commit: a07b9687d21ebbd963a7aea28edd7850daf79411
Parents: 9bd7642
Author: Zach York 
Authored: Thu Feb 2 02:44:58 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 16:11:53 2017 -0800

--
 .../apache/hadoop/hbase/regionserver/RSRpcServices.java | 12 
 1 file changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a07b9687/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index f28e726..46a8120 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.BindException;
@@ -2662,6 +2663,17 @@ public class RSRpcServices implements 
HBaseRPCErrorHandler,
   // row that the client has last seen.
   closeScanner(region, scanner, scannerName);
 
+  // rethrow DoNotRetryIOException. This can avoid the retry in 
ClientScanner.
+  if (e instanceof DoNotRetryIOException) {
+throw e;
+  }
+
+  // If it is a FileNotFoundException, wrap as a
+  // DoNotRetryIOException. This can avoid the retry in ClientScanner.
+  if (e instanceof FileNotFoundException) {
+throw new DoNotRetryIOException(e);
+  }
+
   // We closed the scanner already. Instead of throwing the 
IOException, and client
   // retrying with the same scannerId only to get USE on the next RPC, 
we directly throw
   // a special exception to save an RPC.

hbase git commit: HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the application

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 d53fafdee -> 30b229f24


HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the 
application


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/30b229f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/30b229f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/30b229f2

Branch: refs/heads/branch-1.3
Commit: 30b229f24edda06119310870c12900d55af9c0f7
Parents: d53fafd
Author: Enis Soztutar 
Authored: Mon Feb 6 12:00:05 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 15:06:58 2017 -0800

--
 .../hadoop/hbase/client/ClientScanner.java  | 24 +--
 .../hbase/regionserver/RSRpcServices.java   | 12 +++-
 .../hadoop/hbase/client/TestFromClientSide.java | 76 +++-
 3 files changed, 105 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/30b229f2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 2b5d3ef..ecaab99 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import com.google.common.annotations.VisibleForTesting;
-
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
@@ -50,6 +48,8 @@ import 
org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Implements the scanner interface for the HBase client. If there are 
multiple regions in a table,
  * this scanner will iterate through them all.
@@ -421,6 +421,9 @@ public class ClientScanner extends AbstractClientScanner {
 // This flag is set when we want to skip the result returned. We do
 // this when we reset scanner because it split under us.
 boolean retryAfterOutOfOrderException = true;
+// Even if we are retrying due to UnknownScannerException, 
ScannerResetException, etc. we should
+// make sure that we are not retrying indefinitely.
+int retriesLeft = getRetries();
 for (;;) {
   try {
 // Server returns a null values if scanning is to stop. Else,
@@ -444,8 +447,18 @@ public class ClientScanner extends AbstractClientScanner {
 // An exception was thrown which makes any partial results that we 
were collecting
 // invalid. The scanner will need to be reset to the beginning of a 
row.
 clearPartialResults();
-// DNRIOEs are thrown to make us break out of retries. Some types of 
DNRIOEs want us
-// to reset the scanner and come back in again.
+
+// Unfortunately, DNRIOE is used in two different semantics.
+// (1) The first is to close the client scanner and bubble up the 
exception all the way
+// to the application. This is preferred when the exception is really 
un-recoverable
+// (like CorruptHFileException, etc). Plain DoNotRetryIOException also 
falls into this
+// bucket usually.
+// (2) Second semantics is to close the current region scanner only, 
but continue the
+// client scanner by overriding the exception. This is usually 
UnknownScannerException,
+// OutOfOrderScannerNextException, etc where the region scanner has to 
be closed, but the
+// application-level ClientScanner has to continue without bubbling up 
the exception to
+// the client. See RSRpcServices to see how it throws DNRIOE's.
+// See also: HBASE-16604, HBASE-17187
 
 // If exception is any but the list below throw it back to the client; 
else setup
 // the scanner and retry.
@@ -457,6 +470,9 @@ public class ClientScanner extends AbstractClientScanner {
 e instanceof ScannerResetException) {
   // Pass. It is easier writing the if loop test as list of what is 
allowed rather than
   // as a list of what is not allowed... so if in here, it means we do 
not throw.
+  if (retriesLeft-- <= 0) {
+throw e; // no more retries
+  }
 } else {
   throw e;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/30b229f2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

hbase git commit: HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the application

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 b123313f1 -> 7cac85b66


HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the 
application


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cac85b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cac85b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cac85b6

Branch: refs/heads/branch-1
Commit: 7cac85b6673acc4547c8ad313674fa65a85e8cbd
Parents: b123313
Author: Enis Soztutar 
Authored: Mon Feb 6 12:00:05 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 13:39:09 2017 -0800

--
 .../hadoop/hbase/client/ClientScanner.java  | 33 +++--
 .../hbase/regionserver/RSRpcServices.java   | 12 ++-
 .../hadoop/hbase/client/TestFromClientSide.java | 77 +++-
 3 files changed, 113 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7cac85b6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 40b5002..52027ff 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -17,7 +17,9 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import static org.apache.hadoop.hbase.client.ConnectionUtils.*;
+import static 
org.apache.hadoop.hbase.client.ConnectionUtils.createClosestRowAfter;
+import static 
org.apache.hadoop.hbase.client.ConnectionUtils.createClosestRowBefore;
+
 import com.google.common.annotations.VisibleForTesting;
 
 import java.io.IOException;
@@ -53,6 +55,8 @@ import 
org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Implements the scanner interface for the HBase client. If there are 
multiple regions in a table,
  * this scanner will iterate through them all.
@@ -398,22 +402,33 @@ public class ClientScanner extends AbstractClientScanner {
   }
 
   private Result[] nextScannerWithRetries(int nbRows) throws IOException {
+int retriesLeft = getRetries();
 for (;;) {
   try {
 return nextScanner(nbRows);
   } catch (DoNotRetryIOException e) {
-handleScanError(e, null);
+handleScanError(e, null, retriesLeft--);
   }
 }
   }
 
   private void handleScanError(DoNotRetryIOException e,
-  MutableBoolean retryAfterOutOfOrderException) throws 
DoNotRetryIOException {
+  MutableBoolean retryAfterOutOfOrderException, int retriesLeft) throws 
DoNotRetryIOException {
 // An exception was thrown which makes any partial results that we were 
collecting
 // invalid. The scanner will need to be reset to the beginning of a row.
 clearPartialResults();
-// DNRIOEs are thrown to make us break out of retries. Some types of 
DNRIOEs want us
-// to reset the scanner and come back in again.
+
+// Unfortunately, DNRIOE is used in two different semantics.
+// (1) The first is to close the client scanner and bubble up the 
exception all the way
+// to the application. This is preferred when the exception is really 
un-recoverable
+// (like CorruptHFileException, etc). Plain DoNotRetryIOException also 
falls into this
+// bucket usually.
+// (2) Second semantics is to close the current region scanner only, but 
continue the
+// client scanner by overriding the exception. This is usually 
UnknownScannerException,
+// OutOfOrderScannerNextException, etc where the region scanner has to be 
closed, but the
+// application-level ClientScanner has to continue without bubbling up the 
exception to
+// the client. See RSRpcServices to see how it throws DNRIOE's.
+// See also: HBASE-16604, HBASE-17187
 
 // If exception is any but the list below throw it back to the client; 
else setup
 // the scanner and retry.
@@ -424,6 +439,9 @@ public class ClientScanner extends AbstractClientScanner {
 e instanceof ScannerResetException) {
   // Pass. It is easier writing the if loop test as list of what is 
allowed rather than
   // as a list of what is not allowed... so if in here, it means we do not 
throw.
+  if (retriesLeft <= 0) {
+throw e; // no more retries
+  }
 } else {
   throw e;
 }
@@ -486,6 +504,9 @@ public class ClientScanner extends AbstractClientScanner {
 // This flag is set when we want to 

hbase git commit: HBASE-17350 Fixup of regionserver group-based assignment

[stack]

Repository: hbase
Updated Branches:
  refs/heads/master 41be3bc2c -> 9ec0ec492


HBASE-17350 Fixup of regionserver group-based assignment

Renamed move_rsgroup_servers as move_servers_rsgroup
Renamed move_rsgroup_tables as move_tables_rsgroup

Minor changes to help text in rsgroup commands making them all same.
Made LOG from RSGroupAdminServer all talk of 'rsgroup' rather than
'group' to be consistent.

Fix for table.jsp where it would fail to display regions because no
type for the protobuf record specified.

Fix it so that move of an offline server to 'default' rsgroup is like
moving the reference to the server to trash (keeps the 'default' group
consistently 'dynamic' regards its server-list).

Fixed another issue where we were stuck in a loop because regions
were in FAILED_OPEN state because no server to assign too so we'd
never recover (a vagary of the current state of Master assignement
but no less a possibility in real world deploys).

Make it so servers are sorted when we list them; its what operator
would expect.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9ec0ec49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9ec0ec49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9ec0ec49

Branch: refs/heads/master
Commit: 9ec0ec49e55169d97c53bddd674f6dac8a52
Parents: 41be3bc
Author: Michael Stack 
Authored: Thu Feb 2 14:21:04 2017 -0800
Committer: Michael Stack 
Committed: Mon Feb 6 13:09:57 2017 -0800

--
 .../hadoop/hbase/rsgroup/RSGroupInfo.java   |  24 +--
 .../apache/hadoop/hbase/util/Addressing.java|  22 +++
 .../hadoop/hbase/util/TestAddressing.java   |  39 +
 .../hbase/rsgroup/RSGroupAdminServer.java   | 153 ++-
 .../hbase/rsgroup/RSGroupInfoManager.java   |   4 +-
 .../hbase/rsgroup/RSGroupInfoManagerImpl.java   |  42 +++--
 .../apache/hadoop/hbase/rsgroup/Utility.java|  48 ++
 .../hadoop/hbase/rsgroup/TestRSGroupsBase.java  |   2 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   3 +-
 .../hadoop/hbase/master/ServerManager.java  |   1 -
 .../hbase/regionserver/RSRpcServices.java   |   3 +-
 .../resources/hbase-webapps/master/table.jsp|  25 ++-
 hbase-shell/src/main/ruby/shell.rb  |   7 +-
 hbase-shell/src/main/ruby/shell/commands.rb |   1 -
 .../src/main/ruby/shell/commands/add_rsgroup.rb |   3 +-
 .../main/ruby/shell/commands/balance_rsgroup.rb |   5 +-
 .../src/main/ruby/shell/commands/get_rsgroup.rb |   5 +-
 .../ruby/shell/commands/get_server_rsgroup.rb   |   5 +-
 .../ruby/shell/commands/get_table_rsgroup.rb|   5 +-
 .../main/ruby/shell/commands/list_procedures.rb |   2 +-
 .../main/ruby/shell/commands/list_rsgroups.rb   |   3 +-
 .../ruby/shell/commands/move_rsgroup_servers.rb |  37 -
 .../ruby/shell/commands/move_rsgroup_tables.rb  |  37 -
 .../ruby/shell/commands/move_servers_rsgroup.rb |  40 +
 .../ruby/shell/commands/move_tables_rsgroup.rb  |  40 +
 .../main/ruby/shell/commands/remove_rsgroup.rb  |   3 +-
 .../src/test/ruby/shell/rsgroup_shell_test.rb   |   4 +-
 27 files changed, 360 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9ec0ec49/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
index e68260a..c4bb157 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
@@ -20,16 +20,19 @@
 
 package org.apache.hadoop.hbase.rsgroup;
 
-import com.google.common.collect.Sets;
-import com.google.common.net.HostAndPort;
-
 import java.util.Collection;
 import java.util.NavigableSet;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Addressing;
+
+import com.google.common.collect.Sets;
+import com.google.common.net.HostAndPort;
 
 /**
  * Stores the group information of region server groups.
@@ -42,25 +45,24 @@ public class RSGroupInfo {
   public static final String NAMESPACEDESC_PROP_GROUP = "hbase.rsgroup.name";
 
   private String name;
-  private Set servers;
+  private SortedSet servers;
   private NavigableSet tables;
 
   public RSGroupInfo(String name) {
-this(name, Sets.newHashSet(), Sets.newTreeSet());
+this(name, Sets.newHashSet(), Sets.newTreeSet());
   }
 
  

hbase git commit: HBASE-17578 Thrift metrics should handle exceptions

[garyh]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 f1dd7bbfb -> d53fafdee


HBASE-17578 Thrift metrics should handle exceptions


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d53fafde
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d53fafde
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d53fafde

Branch: refs/heads/branch-1.3
Commit: d53fafdeedafcf8cc5ed636f61af65e6e09f635d
Parents: f1dd7bb
Author: Gary Helmling 
Authored: Wed Dec 28 13:13:24 2016 -0800
Committer: Gary Helmling 
Committed: Mon Feb 6 12:30:30 2017 -0800

--
 .../hbase/ipc/MetricsHBaseServerSource.java |  37 +-
 .../hbase/metrics/ExceptionTrackingSource.java  |  56 +
 .../hbase/thrift/MetricsThriftServerSource.java |   5 +-
 .../hbase/ipc/MetricsHBaseServerSourceImpl.java |  85 +
 .../metrics/ExceptionTrackingSourceImpl.java| 118 +++
 .../thrift/MetricsThriftServerSourceImpl.java   |   4 +-
 .../hbase/thrift/HbaseHandlerMetricsProxy.java  |   9 +-
 .../hadoop/hbase/thrift/ThriftMetrics.java  |  61 ++
 .../hadoop/hbase/thrift/ThriftServerRunner.java | 102 +++-
 .../thrift2/ThriftHBaseServiceHandler.java  |  46 +++-
 .../hbase/thrift/ErrorThrowingGetObserver.java  | 102 
 .../hadoop/hbase/thrift/TestThriftServer.java   |  79 -
 .../thrift2/TestThriftHBaseServiceHandler.java  |  71 ++-
 13 files changed, 611 insertions(+), 164 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d53fafde/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
index ac14bd8..caa12a0 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSource.java
@@ -19,9 +19,9 @@
 
 package org.apache.hadoop.hbase.ipc;
 
-import org.apache.hadoop.hbase.metrics.BaseSource;
+import org.apache.hadoop.hbase.metrics.ExceptionTrackingSource;
 
-public interface MetricsHBaseServerSource extends BaseSource {
+public interface MetricsHBaseServerSource extends ExceptionTrackingSource {
   String AUTHORIZATION_SUCCESSES_NAME = "authorizationSuccesses";
   String AUTHORIZATION_SUCCESSES_DESC =
   "Number of authorization successes.";
@@ -71,22 +71,6 @@ public interface MetricsHBaseServerSource extends BaseSource 
{
   String NUM_LIFO_MODE_SWITCHES_DESC = "Total number of calls in general queue 
which " +
 "were served from the tail of the queue";
 
-  String EXCEPTIONS_NAME="exceptions";
-  String EXCEPTIONS_DESC="Exceptions caused by requests";
-  String EXCEPTIONS_TYPE_DESC="Number of requests that resulted in the 
specified type of Exception";
-  String EXCEPTIONS_OOO_NAME="exceptions.OutOfOrderScannerNextException";
-  String EXCEPTIONS_BUSY_NAME="exceptions.RegionTooBusyException";
-  String EXCEPTIONS_UNKNOWN_NAME="exceptions.UnknownScannerException";
-  String EXCEPTIONS_SCANNER_RESET_NAME="exceptions.ScannerResetException";
-  String EXCEPTIONS_SANITY_NAME="exceptions.FailedSanityCheckException";
-  String EXCEPTIONS_MOVED_NAME="exceptions.RegionMovedException";
-  String EXCEPTIONS_NSRE_NAME="exceptions.NotServingRegionException";
-  String EXCEPTIONS_MULTI_TOO_LARGE_NAME = "exceptions.multiResponseTooLarge";
-  String EXCEPTIONS_MULTI_TOO_LARGE_DESC = "A response to a multi request was 
too large and the " +
-  "rest of the requests will have to be retried.";
-  String EXCEPTIONS_CALL_QUEUE_TOO_BIG = "exceptions.callQueueTooBig";
-  String EXCEPTIONS_CALL_QUEUE_TOO_BIG_DESC = "Call queue is full";
-
   void authorizationSuccess();
 
   void authorizationFailure();
@@ -97,21 +81,6 @@ public interface MetricsHBaseServerSource extends BaseSource 
{
 
   void authenticationFallback();
 
-  void exception();
-
-  /**
-   * Different types of exceptions
-   */
-  void outOfOrderException();
-  void failedSanityException();
-  void movedRegionException();
-  void notServingRegionException();
-  void unknownScannerException();
-  void scannerResetException();
-  void tooBusyException();
-  void multiActionTooLargeException();
-  void callQueueTooBigException();
-
   void sentBytes(long count);
 
   void receivedBytes(int count);
@@ -125,6 +94,4 @@ public interface MetricsHBaseServerSource extends BaseSource 
{
   void processedCall(int processingTime);
 
   void queuedAndProcessedCall(int totalTime);
-
-
 }

hbase git commit: HBASE-17593 Update hadoop 3 version to 3.0.0-alpha2.

[appy]

Repository: hbase
Updated Branches:
  refs/heads/master 4d730244a -> 41be3bc2c


HBASE-17593 Update hadoop 3 version to 3.0.0-alpha2.

Change-Id: Ic5411229aaab3d645dc6751ecfbbda1b2d93a4b9


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41be3bc2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41be3bc2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41be3bc2

Branch: refs/heads/master
Commit: 41be3bc2cc42565335d55f553c536b1fce2aa023
Parents: 4d73024
Author: Apekshit Sharma 
Authored: Fri Feb 3 14:31:25 2017 -0800
Committer: Apekshit Sharma 
Committed: Mon Feb 6 12:18:21 2017 -0800

--
 dev-support/hbase-personality.sh | 2 +-
 pom.xml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/41be3bc2/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 6c9d566..6aa95f9 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -54,7 +54,7 @@ function personality_globals
   # TODO use PATCH_BRANCH to select hadoop versions to use.
   # All supported Hadoop versions that we want to test the compilation with
   HBASE_MASTER_HADOOP2_VERSIONS="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 
2.7.3"
-  HBASE_MASTER_HADOOP3_VERSIONS="3.0.0-alpha1"
+  HBASE_MASTER_HADOOP3_VERSIONS="3.0.0-alpha2"
 
   HBASE_HADOOP2_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 
2.6.4 2.6.5 2.7.1 2.7.2 2.7.3"
   HBASE_HADOOP3_VERSIONS=""

http://git-wip-us.apache.org/repos/asf/hbase/blob/41be3bc2/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d837d6..535573d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1212,7 +1212,7 @@
 ${compileSource}
 
 2.7.1
-3.0.0-alpha1
+3.0.0-alpha2
 

hbase git commit: HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the application

[enis]

Repository: hbase
Updated Branches:
  refs/heads/master 7a6518f05 -> 4d730244a


HBASE-17187 DoNotRetryExceptions from coprocessors should bubble up to the 
application


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4d730244
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4d730244
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4d730244

Branch: refs/heads/master
Commit: 4d730244a98afac3769b1232e49afa27eece7d81
Parents: 7a6518f
Author: Enis Soztutar 
Authored: Mon Feb 6 12:00:05 2017 -0800
Committer: Enis Soztutar 
Committed: Mon Feb 6 12:00:42 2017 -0800

--
 .../hadoop/hbase/client/ClientScanner.java  | 31 ++--
 .../hbase/regionserver/RSRpcServices.java   | 14 +++-
 .../hadoop/hbase/client/TestFromClientSide.java | 79 +++-
 3 files changed, 112 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4d730244/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index 313cb63..57a7702 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -21,8 +21,6 @@ import static 
org.apache.hadoop.hbase.client.ConnectionUtils.calcEstimatedSize;
 import static 
org.apache.hadoop.hbase.client.ConnectionUtils.createClosestRowAfter;
 import static 
org.apache.hadoop.hbase.client.ConnectionUtils.createClosestRowBefore;
 
-import com.google.common.annotations.VisibleForTesting;
-
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
@@ -55,6 +53,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MapReduceProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Implements the scanner interface for the HBase client. If there are 
multiple regions in a table,
  * this scanner will iterate through them all.
@@ -395,22 +395,33 @@ public abstract class ClientScanner extends 
AbstractClientScanner {
   }
 
   private Result[] nextScannerWithRetries(int nbRows) throws IOException {
+int retriesLeft = getRetries();
 for (;;) {
   try {
 return nextScanner(nbRows);
   } catch (DoNotRetryIOException e) {
-handleScanError(e, null);
+handleScanError(e, null, retriesLeft--);
   }
 }
   }
 
   private void handleScanError(DoNotRetryIOException e,
-  MutableBoolean retryAfterOutOfOrderException) throws 
DoNotRetryIOException {
+  MutableBoolean retryAfterOutOfOrderException, int retriesLeft) throws 
DoNotRetryIOException {
 // An exception was thrown which makes any partial results that we were 
collecting
 // invalid. The scanner will need to be reset to the beginning of a row.
 clearPartialResults();
-// DNRIOEs are thrown to make us break out of retries. Some types of 
DNRIOEs want us
-// to reset the scanner and come back in again.
+
+// Unfortunately, DNRIOE is used in two different semantics.
+// (1) The first is to close the client scanner and bubble up the 
exception all the way
+// to the application. This is preferred when the exception is really 
un-recoverable
+// (like CorruptHFileException, etc). Plain DoNotRetryIOException also 
falls into this
+// bucket usually.
+// (2) Second semantics is to close the current region scanner only, but 
continue the
+// client scanner by overriding the exception. This is usually 
UnknownScannerException,
+// OutOfOrderScannerNextException, etc where the region scanner has to be 
closed, but the
+// application-level ClientScanner has to continue without bubbling up the 
exception to
+// the client. See RSRpcServices to see how it throws DNRIOE's.
+// See also: HBASE-16604, HBASE-17187
 
 // If exception is any but the list below throw it back to the client; 
else setup
 // the scanner and retry.
@@ -421,6 +432,9 @@ public abstract class ClientScanner extends 
AbstractClientScanner {
 e instanceof ScannerResetException) {
   // Pass. It is easier writing the if loop test as list of what is 
allowed rather than
   // as a list of what is not allowed... so if in here, it means we do not 
throw.
+  if (retriesLeft <= 0) {
+throw e; // no more retries
+  }
 } else {
   throw e;
 }
@@ -483,6 +497,9 @@ public abstract class ClientScanner extends 

[2/2] hbase git commit: HBASE-16999 Implement master and regionserver synchronization of quota state

[elserj]

HBASE-16999 Implement master and regionserver synchronization of quota state

* Implement the RegionServer reading violation from the quota table
* Implement the Master reporting violations to the quota table
* RegionServers need to track its enforced policies


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/787e6c50
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/787e6c50
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/787e6c50

Branch: refs/heads/HBASE-16961
Commit: 787e6c50af026ae8211b1553339b169f61c6ffac
Parents: a00bd18
Author: Josh Elser 
Authored: Fri Nov 18 15:38:19 2016 -0500
Committer: Josh Elser 
Committed: Mon Feb 6 13:56:40 2017 -0500

--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  92 -
 .../org/apache/hadoop/hbase/master/HMaster.java |  35 +++-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |   5 +-
 .../hbase/quotas/RegionServerQuotaManager.java  | 200 ---
 .../quotas/RegionServerRpcQuotaManager.java | 200 +++
 .../quotas/RegionServerSpaceQuotaManager.java   | 169 
 .../quotas/SpaceQuotaViolationNotifier.java |  16 +-
 .../SpaceQuotaViolationNotifierFactory.java |  62 ++
 .../SpaceQuotaViolationNotifierForTest.java |   4 +
 ...SpaceQuotaViolationPolicyRefresherChore.java | 154 ++
 .../TableSpaceQuotaViolationNotifier.java   |  55 +
 .../hbase/regionserver/HRegionServer.java   |  21 +-
 .../hbase/regionserver/RSRpcServices.java   |   7 +-
 .../regionserver/RegionServerServices.java  |  12 +-
 .../hadoop/hbase/MockRegionServerServices.java  |  10 +-
 .../hadoop/hbase/master/MockRegionServer.java   |  10 +-
 .../TestQuotaObserverChoreWithMiniCluster.java  |   2 +
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |  49 +
 .../hadoop/hbase/quotas/TestQuotaThrottle.java  |   4 +-
 .../TestRegionServerSpaceQuotaManager.java  | 127 
 ...SpaceQuotaViolationPolicyRefresherChore.java | 131 
 .../TestTableSpaceQuotaViolationNotifier.java   | 144 +
 22 files changed, 1283 insertions(+), 226 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/787e6c50/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 1640ddc..505e94b 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -23,16 +23,20 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
@@ -43,7 +47,12 @@ import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
+import 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Strings;
 
@@ -52,9 +61,8 @@ import org.apache.hadoop.hbase.util.Strings;
  * 
  * ROW-KEY  FAM/QUALDATA
  *   n.namespace q:s global-quotas
- *   n.namespace u:dusize in bytes
  *   t.table q:s global-quotas
- *   t.table u:dusize in bytes
+ *   t.table u:vspace violation policy
  *   u.user  q:s 

hbase git commit: HBASE-17601 close() in TableRecordReaderImpl assumes the split has started (Michael Axiak)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 7efa34b42 -> 9bd76426f


HBASE-17601 close() in TableRecordReaderImpl assumes the split has started 
(Michael Axiak)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9bd76426
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9bd76426
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9bd76426

Branch: refs/heads/branch-1.2
Commit: 9bd76426f427738438703741c0bba167757688ce
Parents: 7efa34b
Author: tedyu 
Authored: Mon Feb 6 11:01:55 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 11:01:55 2017 -0800

--
 .../org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java| 4 +++-
 .../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9bd76426/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index 6356106..2ace8e2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -154,7 +154,9 @@ public class TableRecordReaderImpl {
   }
 
   public void close() {
-this.scanner.close();
+if (this.scanner != null) {
+  this.scanner.close();
+}
 try {
   this.htable.close();
 } catch (IOException ioe) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9bd76426/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index dcc4943..6f1d140 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -161,7 +161,9 @@ public class TableRecordReaderImpl {
*
*/
   public void close() {
-this.scanner.close();
+if (this.scanner != null) {
+  this.scanner.close();
+}
 try {
   this.htable.close();
 } catch (IOException ioe) {

hbase git commit: HBASE-17601 close() in TableRecordReaderImpl assumes the split has started (Michael Axiak)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 4456d2285 -> 35e4eb7a7


HBASE-17601 close() in TableRecordReaderImpl assumes the split has started 
(Michael Axiak)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/35e4eb7a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/35e4eb7a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/35e4eb7a

Branch: refs/heads/branch-1
Commit: 35e4eb7a7ab8055825e3c35f13521d5e9e0261e3
Parents: 4456d22
Author: tedyu 
Authored: Mon Feb 6 10:33:05 2017 -0800
Committer: tedyu 
Committed: Mon Feb 6 10:33:05 2017 -0800

--
 .../org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java| 4 +++-
 .../org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/35e4eb7a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
index 6356106..2ace8e2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
@@ -154,7 +154,9 @@ public class TableRecordReaderImpl {
   }
 
   public void close() {
-this.scanner.close();
+if (this.scanner != null) {
+  this.scanner.close();
+}
 try {
   this.htable.close();
 } catch (IOException ioe) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/35e4eb7a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index dcc4943..6f1d140 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -161,7 +161,9 @@ public class TableRecordReaderImpl {
*
*/
   public void close() {
-this.scanner.close();
+if (this.scanner != null) {
+  this.scanner.close();
+}
 try {
   this.htable.close();
 } catch (IOException ioe) {