[hbase] 01/02: Revert "HBASE-27091 Speed up the loading of table descriptor from filesystem (#4493)"

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit 86b7b027b774fdac1f3c93707fcf32564b6ba588
Author: Yu Li 
AuthorDate: Wed Jun 15 00:47:28 2022 +0800

Revert "HBASE-27091 Speed up the loading of table descriptor from 
filesystem (#4493)"

This reverts commit 5d0f4dc729a471c1dd0b926f0d6fabdbca01d7f7 to correct 
signed-off info.
---
 .../org/apache/hadoop/hbase/HBaseServerBase.java   | 15 +---
 .../org/apache/hadoop/hbase/TableDescriptors.java  |  8 +--
 .../org/apache/hadoop/hbase/master/HMaster.java|  1 -
 .../hadoop/hbase/regionserver/HRegionServer.java   |  1 -
 .../hadoop/hbase/util/FSTableDescriptors.java  | 80 +++---
 .../hadoop/hbase/util/TestFSTableDescriptors.java  | 31 -
 6 files changed, 12 insertions(+), 124 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
index e148c7da84..f39e260914 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
@@ -231,10 +231,8 @@ public abstract class HBaseServerBase> extends
 // init the filesystem
 this.dataFs = new HFileSystem(this.conf, useHBaseChecksum);
 this.dataRootDir = CommonFSUtils.getRootDir(this.conf);
-int tableDescriptorParallelLoadThreads =
-  conf.getInt("hbase.tabledescriptor.parallel.load.threads", 0);
 this.tableDescriptors = new FSTableDescriptors(this.dataFs, 
this.dataRootDir,
-  !canUpdateTableDescriptor(), cacheTableDescriptor(), 
tableDescriptorParallelLoadThreads);
+  !canUpdateTableDescriptor(), cacheTableDescriptor());
   }
 
   public HBaseServerBase(Configuration conf, String name) throws IOException {
@@ -468,17 +466,6 @@ public abstract class HBaseServerBase> extends
 }
   }
 
-  protected final void closeTableDescriptors() {
-if (this.tableDescriptors != null) {
-  LOG.info("Close table descriptors");
-  try {
-this.tableDescriptors.close();
-  } catch (IOException e) {
-LOG.debug("Failed to close table descriptors gracefully", e);
-  }
-}
-  }
-
   /**
* In order to register ShutdownHook, this method is called when HMaster and 
HRegionServer are
* started. For details, please refer to HBASE-26951
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index 9ecdf39679..1dc17eff0d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -27,7 +26,7 @@ import org.apache.yetus.audience.InterfaceAudience;
  * Get, remove and modify table descriptors.
  */
 @InterfaceAudience.Private
-public interface TableDescriptors extends Closeable {
+public interface TableDescriptors {
 
   /**
* Test whether a given table exists, i.e, has a table descriptor.
@@ -36,11 +35,6 @@ public interface TableDescriptors extends Closeable {
 return get(tableName) != null;
   }
 
-  @Override
-  default void close() throws IOException {
-// do nothing by default
-  }
-
   /**
* @return TableDescriptor for tablename
*/
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 2b818d9cc2..c9556a80ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -601,7 +601,6 @@ public class HMaster extends 
HBaseServerBase implements Maste
   this.rpcServices.stop();
 }
 closeZooKeeper();
-closeTableDescriptors();
 span.setStatus(StatusCode.OK);
   } finally {
 span.end();
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1865929dc5..0ee4c5d746 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -977,7 +977,6 @@ public class HRegionServer extends 
HBaseServerBase
   ZNodeClearer.deleteMyEphemeralNodeOnDisk();
 
   closeZooKeeper();
-  closeTableDescriptors();
   LOG.info("Exiting; stopping=" + this.serverName + "; zookeeper 
connection closed.");
  

[hbase] branch master updated (5d0f4dc729 -> c6298c709a)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 5d0f4dc729 HBASE-27091 Speed up the loading of table descriptor from 
filesystem (#4493)
 new 86b7b027b7 Revert "HBASE-27091 Speed up the loading of table 
descriptor from filesystem (#4493)"
 new c6298c709a HBASE-27091 Speed up the loading of table descriptor from 
filesystem (#4493)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:

[hbase] 02/02: HBASE-27091 Speed up the loading of table descriptor from filesystem (#4493)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit c6298c709a10e581f20a5038f2cb2463f01b7195
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Sat Jun 11 08:48:23 2022 +0800

HBASE-27091 Speed up the loading of table descriptor from filesystem (#4493)

Signed-off-by: Huaxiang Sun 
Signed-off-by: Yu Li 
---
 .../org/apache/hadoop/hbase/HBaseServerBase.java   | 15 +++-
 .../org/apache/hadoop/hbase/TableDescriptors.java  |  8 ++-
 .../org/apache/hadoop/hbase/master/HMaster.java|  1 +
 .../hadoop/hbase/regionserver/HRegionServer.java   |  1 +
 .../hadoop/hbase/util/FSTableDescriptors.java  | 80 +++---
 .../hadoop/hbase/util/TestFSTableDescriptors.java  | 31 +
 6 files changed, 124 insertions(+), 12 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
index f39e260914..e148c7da84 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
@@ -231,8 +231,10 @@ public abstract class HBaseServerBase> extends
 // init the filesystem
 this.dataFs = new HFileSystem(this.conf, useHBaseChecksum);
 this.dataRootDir = CommonFSUtils.getRootDir(this.conf);
+int tableDescriptorParallelLoadThreads =
+  conf.getInt("hbase.tabledescriptor.parallel.load.threads", 0);
 this.tableDescriptors = new FSTableDescriptors(this.dataFs, 
this.dataRootDir,
-  !canUpdateTableDescriptor(), cacheTableDescriptor());
+  !canUpdateTableDescriptor(), cacheTableDescriptor(), 
tableDescriptorParallelLoadThreads);
   }
 
   public HBaseServerBase(Configuration conf, String name) throws IOException {
@@ -466,6 +468,17 @@ public abstract class HBaseServerBase> extends
 }
   }
 
+  protected final void closeTableDescriptors() {
+if (this.tableDescriptors != null) {
+  LOG.info("Close table descriptors");
+  try {
+this.tableDescriptors.close();
+  } catch (IOException e) {
+LOG.debug("Failed to close table descriptors gracefully", e);
+  }
+}
+  }
+
   /**
* In order to register ShutdownHook, this method is called when HMaster and 
HRegionServer are
* started. For details, please refer to HBASE-26951
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index 1dc17eff0d..9ecdf39679 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -26,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
  * Get, remove and modify table descriptors.
  */
 @InterfaceAudience.Private
-public interface TableDescriptors {
+public interface TableDescriptors extends Closeable {
 
   /**
* Test whether a given table exists, i.e, has a table descriptor.
@@ -35,6 +36,11 @@ public interface TableDescriptors {
 return get(tableName) != null;
   }
 
+  @Override
+  default void close() throws IOException {
+// do nothing by default
+  }
+
   /**
* @return TableDescriptor for tablename
*/
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c9556a80ed..2b818d9cc2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -601,6 +601,7 @@ public class HMaster extends 
HBaseServerBase implements Maste
   this.rpcServices.stop();
 }
 closeZooKeeper();
+closeTableDescriptors();
 span.setStatus(StatusCode.OK);
   } finally {
 span.end();
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 0ee4c5d746..1865929dc5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -977,6 +977,7 @@ public class HRegionServer extends 
HBaseServerBase
   ZNodeClearer.deleteMyEphemeralNodeOnDisk();
 
   closeZooKeeper();
+  closeTableDescriptors();
   LOG.info("Exiting; stopping=" + this.serverName + "; zookeeper 
connection closed.");
   span.setStatus(StatusCode.OK);
 }

[hbase] branch master updated: HBASE-27091 Speed up the loading of table descriptor from filesystem (#4493)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 5d0f4dc729 HBASE-27091 Speed up the loading of table descriptor from 
filesystem (#4493)
5d0f4dc729 is described below

commit 5d0f4dc729a471c1dd0b926f0d6fabdbca01d7f7
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Sat Jun 11 08:48:23 2022 +0800

HBASE-27091 Speed up the loading of table descriptor from filesystem (#4493)

Signed-off-by: Yu Li 
---
 .../org/apache/hadoop/hbase/HBaseServerBase.java   | 15 +++-
 .../org/apache/hadoop/hbase/TableDescriptors.java  |  8 ++-
 .../org/apache/hadoop/hbase/master/HMaster.java|  1 +
 .../hadoop/hbase/regionserver/HRegionServer.java   |  1 +
 .../hadoop/hbase/util/FSTableDescriptors.java  | 80 +++---
 .../hadoop/hbase/util/TestFSTableDescriptors.java  | 31 +
 6 files changed, 124 insertions(+), 12 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
index f39e260914..e148c7da84 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
@@ -231,8 +231,10 @@ public abstract class HBaseServerBase> extends
 // init the filesystem
 this.dataFs = new HFileSystem(this.conf, useHBaseChecksum);
 this.dataRootDir = CommonFSUtils.getRootDir(this.conf);
+int tableDescriptorParallelLoadThreads =
+  conf.getInt("hbase.tabledescriptor.parallel.load.threads", 0);
 this.tableDescriptors = new FSTableDescriptors(this.dataFs, 
this.dataRootDir,
-  !canUpdateTableDescriptor(), cacheTableDescriptor());
+  !canUpdateTableDescriptor(), cacheTableDescriptor(), 
tableDescriptorParallelLoadThreads);
   }
 
   public HBaseServerBase(Configuration conf, String name) throws IOException {
@@ -466,6 +468,17 @@ public abstract class HBaseServerBase> extends
 }
   }
 
+  protected final void closeTableDescriptors() {
+if (this.tableDescriptors != null) {
+  LOG.info("Close table descriptors");
+  try {
+this.tableDescriptors.close();
+  } catch (IOException e) {
+LOG.debug("Failed to close table descriptors gracefully", e);
+  }
+}
+  }
+
   /**
* In order to register ShutdownHook, this method is called when HMaster and 
HRegionServer are
* started. For details, please refer to HBASE-26951
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index 1dc17eff0d..9ecdf39679 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -26,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
  * Get, remove and modify table descriptors.
  */
 @InterfaceAudience.Private
-public interface TableDescriptors {
+public interface TableDescriptors extends Closeable {
 
   /**
* Test whether a given table exists, i.e, has a table descriptor.
@@ -35,6 +36,11 @@ public interface TableDescriptors {
 return get(tableName) != null;
   }
 
+  @Override
+  default void close() throws IOException {
+// do nothing by default
+  }
+
   /**
* @return TableDescriptor for tablename
*/
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c9556a80ed..2b818d9cc2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -601,6 +601,7 @@ public class HMaster extends 
HBaseServerBase implements Maste
   this.rpcServices.stop();
 }
 closeZooKeeper();
+closeTableDescriptors();
 span.setStatus(StatusCode.OK);
   } finally {
 span.end();
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 0ee4c5d746..1865929dc5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -977,6 +977,7 @@ public class HRegionServer extends 
HBaseServerBase
   ZNodeClearer.deleteMyEphemeralNodeOnDisk();
 
   closeZooKeeper();
+  closeTableDescrip

[hbase] branch branch-1 updated: HBASE-26977 HMaster's ShutdownHook does not take effect, if tablesOnMaster is false (#4377)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-1 by this push:
 new d4b9fcec1b4 HBASE-26977 HMaster's ShutdownHook does not take effect, 
if tablesOnMaster is false (#4377)
d4b9fcec1b4 is described below

commit d4b9fcec1b47ae9c5ef48498c3551ad847716b2a
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Mon May 23 23:51:53 2022 +0800

HBASE-26977 HMaster's ShutdownHook does not take effect, if tablesOnMaster 
is false (#4377)

Signed-off-by: Yu Li 
---
 .../hadoop/hbase/regionserver/HRegionServer.java   | 25 +-
 .../org/apache/hadoop/hbase/master/TestMaster.java |  6 ++
 .../regionserver/TestRegionServerNoMaster.java |  6 ++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 0a423d62610..bbe5dbbbfbe 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -320,6 +320,9 @@ public class HRegionServer extends HasThread implements
   // of HRegionServer in isolation.
   private volatile boolean stopped = false;
 
+  // Only for testing
+  private boolean isShutdownHookInstalled = false;
+
   // Go down hard. Used if file system becomes unavailable and also in
   // debugging and unit tests.
   private AtomicBoolean abortRequested;
@@ -1012,7 +1015,12 @@ public class HRegionServer extends HasThread implements
*/
   @Override
   public void run() {
+if (isStopped()) {
+  LOG.info("Skipping run; stopped");
+  return;
+}
 try {
+  installShutdownHook();
   // Do pre-registration initializations; zookeeper, lease threads, etc.
   preRegistrationInitialization();
 } catch (Throwable e) {
@@ -1021,7 +1029,6 @@ public class HRegionServer extends HasThread implements
 
 try {
   if (!isStopped() && !isAborted()) {
-ShutdownHook.install(conf, fs, this, Thread.currentThread());
 // Initialize the RegionServerCoprocessorHost now that our ephemeral
 // node was created, in case any coprocessors want to use ZooKeeper
 this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
@@ -1262,6 +1269,22 @@ public class HRegionServer extends HasThread implements
 LOG.info(Thread.currentThread().getName() + " exiting");
   }
 
+  /**
+   * This method is called when HMaster and HRegionServer are started. Please 
see HBASE-26977
+   * for details.
+   */
+  private void installShutdownHook() {
+ShutdownHook.install(conf, fs, this, Thread.currentThread());
+isShutdownHookInstalled = true;
+  }
+
+  /**
+   * This method is used for testing.
+   */
+  public boolean isShutdownHookInstalled() {
+return isShutdownHookInstalled;
+  }
+
   private boolean containsMetaTableRegions() {
 return 
onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
   }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 39df948adb3..ea257102c44 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -249,5 +249,11 @@ public class TestMaster {
   TEST_UTIL.deleteTable(tableName);
 }
   }
+
+  @Test
+  public void testInstallShutdownHook() {
+// Test for HBASE-26977
+
assertTrue(TEST_UTIL.getMiniHBaseCluster().getMaster().isShutdownHookInstalled());
+  }
 }
 
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
index a419b78aa1c..63298e5dcef 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
@@ -492,4 +492,10 @@ public class TestRegionServerNoMaster {
   openRegion(HTU, getRS(), hri);
 }
   }
+
+  @Test
+  public void testInstallShutdownHook() {
+// Test for HBASE-26977
+
Assert.assertTrue(HTU.getMiniHBaseCluster().getRegionServer(0).isShutdownHookInstalled());
+  }
 }

[hbase] branch master updated: HBASE-27040 Optimize the log display of the ZKProcedureUtil.java (#4434)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new b7876488479 HBASE-27040 Optimize the log display of the 
ZKProcedureUtil.java (#4434)
b7876488479 is described below

commit b78764884797c5c443e19daaa4ffa895cf25
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Tue May 17 15:29:43 2022 +0800

HBASE-27040 Optimize the log display of the ZKProcedureUtil.java (#4434)

Signed-off-by: Yu Li 
---
 .../main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
index 5863da80140..1dff3ced994 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -262,7 +262,7 @@ public abstract class ZKProcedureUtil extends ZKListener 
implements Closeable {
   }
 
   public void clearZNodes(String procedureName) throws KeeperException {
-LOG.info("Clearing all znodes for procedure " + procedureName + "including 
nodes "
+LOG.info("Clearing all znodes for procedure " + procedureName + " 
including nodes "
   + acquiredZnode + " " + reachedZnode + " " + abortZnode);
 
 // Make sure we trigger the watches on these nodes by creating them. 
(HBASE-13885)

[hbase] branch branch-1 updated: HBASE-26998 TestRegionProcessRowsWithLocks#testProcessExceptionAndRollBack is broken in branch-1 (#4404)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-1 by this push:
 new 1599a48687 HBASE-26998 
TestRegionProcessRowsWithLocks#testProcessExceptionAndRollBack is broken in 
branch-1 (#4404)
1599a48687 is described below

commit 1599a48687f696cf48e47f27881a25a569e64884
Author: Yu Li 
AuthorDate: Fri May 6 23:38:44 2022 +0800

HBASE-26998 TestRegionProcessRowsWithLocks#testProcessExceptionAndRollBack 
is broken in branch-1 (#4404)

Signed-off-by: Duo Zhang 
---
 .../hbase/regionserver/TestRegionProcessRowsWithLocks.java| 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionProcessRowsWithLocks.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionProcessRowsWithLocks.java
index e4c38e7245..c2a52d3f03 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionProcessRowsWithLocks.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionProcessRowsWithLocks.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
 import static org.junit.Assert.assertEquals;
 
 import com.google.protobuf.Message;
+import com.google.protobuf.ServiceException;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -112,7 +113,7 @@ public class TestRegionProcessRowsWithLocks {
 util.shutdownMiniCluster();
   }
 
-  public void prepareTestData() throws Exception {
+  public void prepareTestData() throws IOException {
 try {
   util.getHBaseAdmin().disableTable(TABLE);
   util.getHBaseAdmin().deleteTable(TABLE);
@@ -131,7 +132,7 @@ public class TestRegionProcessRowsWithLocks {
   }
 
   @Test
-  public void testProcessNormal() throws Throwable {
+  public void testProcessNormal() throws ServiceException, IOException {
 prepareTestData();
 List regions = util.getHBaseCluster().getRegions(TABLE);
 HRegion region = regions.get(0);
@@ -152,7 +153,7 @@ public class TestRegionProcessRowsWithLocks {
   }
 
   @Test
-  public void testProcessExceptionAndRollBack() throws Throwable {
+  public void testProcessExceptionAndRollBack() throws IOException {
 prepareTestData();
 List regions = util.getHBaseCluster().getRegions(TABLE);
 HRegion region = regions.get(0);
@@ -172,7 +173,7 @@ public class TestRegionProcessRowsWithLocks {
 try {
   incrementCounter(table);
   Assert.fail("Should throw IOException.");
-} catch (IOException e) {
+} catch (ServiceException | IOException e) {
 }
 
 long endMemstoreSize = region.getMemstoreSize();
@@ -184,7 +185,7 @@ public class TestRegionProcessRowsWithLocks {
 (endFlushableSize - startFlushableSize));
   }
 
-  private int incrementCounter(Table table) throws Throwable {
+  private int incrementCounter(Table table) throws ServiceException, 
IOException {
 CoprocessorRpcChannel channel = table.coprocessorService(ROW);
 RowProcessorEndpoint.IncrementCounterProcessor processor =
 new RowProcessorEndpoint.IncrementCounterProcessor(ROW);

[hbase] branch branch-2 updated: HBASE-26977 HMaster's ShutdownHook does not take effect, if tablesOnMaster is false (#4397)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new cb61d6a4bd HBASE-26977 HMaster's ShutdownHook does not take effect, if 
tablesOnMaster is false (#4397)
cb61d6a4bd is described below

commit cb61d6a4bd1a13df6b86dc3af050b988a4a6dc5a
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Tue May 3 18:59:38 2022 +0800

HBASE-26977 HMaster's ShutdownHook does not take effect, if tablesOnMaster 
is false (#4397)

Signed-off-by: Yu Li 
---
 .../hadoop/hbase/regionserver/HRegionServer.java | 20 +++-
 .../org/apache/hadoop/hbase/master/TestMaster.java   |  6 ++
 .../hbase/regionserver/TestRegionServerNoMaster.java |  6 ++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 97388ea186..29a0b50809 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -342,6 +342,8 @@ public class HRegionServer extends Thread
   // shutdown. Also set by call to stop when debugging or running unit tests
   // of HRegionServer in isolation.
   private volatile boolean stopped = false;
+  // Only for testing
+  private boolean isShutdownHookInstalled = false;
 
   // Go down hard. Used if file system becomes unavailable and also in
   // debugging and unit tests.
@@ -1023,6 +1025,7 @@ public class HRegionServer extends Thread
   return;
 }
 try {
+  installShutdownHook();
   // Do pre-registration initializations; zookeeper, lease threads, etc.
   preRegistrationInitialization();
 } catch (Throwable e) {
@@ -1031,7 +1034,6 @@ public class HRegionServer extends Thread
 
 try {
   if (!isStopped() && !isAborted()) {
-ShutdownHook.install(conf, dataFs, this, Thread.currentThread());
 // Initialize the RegionServerCoprocessorHost now that our ephemeral
 // node was created, in case any coprocessors want to use ZooKeeper
 this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
@@ -1254,6 +1256,22 @@ public class HRegionServer extends Thread
 LOG.info("Exiting; stopping=" + this.serverName + "; zookeeper connection 
closed.");
   }
 
+  /**
+   * This method is called when HMaster and HRegionServer are started.
+   * Please see to HBASE-26977 for details.
+   */
+  private void installShutdownHook() {
+ShutdownHook.install(conf, dataFs, this, Thread.currentThread());
+isShutdownHookInstalled = true;
+  }
+
+  /**
+   * This method is used for testing.
+   */
+  public boolean isShutdownHookInstalled() {
+return isShutdownHookInstalled;
+  }
+
   private boolean containsMetaTableRegions() {
 return 
onlineRegions.containsKey(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
   }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index a266e534aa..e78fa081a6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -260,4 +260,10 @@ public class TestMaster {
 // Assert lock gets put in place again.
 assertTrue(fs.exists(hbckLockPath));
   }
+
+  @Test
+  public void testInstallShutdownHook() {
+// Test for HBASE-26977
+
assertTrue(TEST_UTIL.getMiniHBaseCluster().getMaster().isShutdownHookInstalled());
+  }
 }
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
index df5c920359..0a3cf332d1 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
@@ -300,4 +300,10 @@ public class TestRegionServerNoMaster {
   openRegion(HTU, getRS(), hri);
 }
   }
+
+  @Test
+  public void testInstallShutdownHook() {
+// Test for HBASE-26977
+
Assert.assertTrue(HTU.getMiniHBaseCluster().getRegionServer(0).isShutdownHookInstalled());
+  }
 }

[hbase] branch master updated: HBASE-26976 Update related comments after HMaster can load the live RS infos from local region (#4375)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new b0c2832b6ec HBASE-26976 Update related comments after HMaster can load 
the live RS infos from local region (#4375)
b0c2832b6ec is described below

commit b0c2832b6ecdf7a72b219cbf81c2ef70a3097579
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Sat Apr 30 10:53:13 2022 +0800

HBASE-26976 Update related comments after HMaster can load the live RS 
infos from local region (#4375)

Signed-off-by: Duo Zhang 
Signed-off-by: Yu Li 
---
 .../src/main/java/org/apache/hadoop/hbase/master/HMaster.java  | 10 --
 .../org/apache/hadoop/hbase/master/RegionServerTracker.java|  2 +-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 91e18a6f88f..b9fa3294b91 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -865,8 +865,8 @@ public class HMaster extends 
HBaseServerBase implements Maste
* from meta region
* Start region server tracker, construct the online servers set and 
find out dead servers and
* schedule SCP for them. The online servers will be constructed by scanning 
zk, and we will also
-   * scan the wal directory to find out possible live region servers, and the 
differences between
-   * these two sets are the dead servers
+   * scan the wal directory and load from master local region to find out 
possible live region servers,
+   * and the differences between these two sets are the dead servers
* 
* 
* If this is a new deploy, schedule a InitMetaProcedure to initialize 
meta
@@ -941,8 +941,6 @@ public class HMaster extends 
HBaseServerBase implements Maste
   this.splitWALManager = new SplitWALManager(this);
 }
 
-
-
 tryMigrateMetaLocationsFromZooKeeper();
 
 createProcedureExecutor();
@@ -963,8 +961,8 @@ public class HMaster extends 
HBaseServerBase implements Maste
 this.assignmentManager.setupRIT(ritList);
 
 // Start RegionServerTracker with listing of servers found with exiting 
SCPs -- these should
-// be registered in the deadServers set -- and with the list of 
servernames out on the
-// filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let 
SCP figure it out).
+// be registered in the deadServers set -- and the servernames loaded from 
the WAL directory
+// and master local region that COULD BE 'alive'(we'll schedule SCPs for 
each and let SCP figure it out).
 // We also pass dirs that are already 'splitting'... so we can do some 
checks down in tracker.
 // TODO: Generate the splitting and live Set in one pass instead of two as 
we currently do.
 this.regionServerTracker.upgrade(
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 63a3ec19fac..84192a7e94c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -129,7 +129,7 @@ public class RegionServerTracker extends ZKListener {
 // '-SPLITTING'. Each splitting server should have a corresponding SCP. 
Log if not.
 splittingServersFromWALDir.stream().filter(s -> 
!deadServersFromPE.contains(s)).
   forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s));
-// create ServerNode for all possible live servers from wal directory
+// create ServerNode for all possible live servers from wal directory and 
master local region
 liveServersBeforeRestart
 .forEach(sn -> 
server.getAssignmentManager().getRegionStates().getOrCreateServer(sn));
 ServerManager serverManager = server.getServerManager();

[hbase] branch branch-2 updated: HBASE-26943 HMaster page style display confusion (#4369)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new e9c9418d1d1 HBASE-26943 HMaster page style display confusion (#4369)
e9c9418d1d1 is described below

commit e9c9418d1d14a5db3b9f4e391f0f6ed80c3f833e
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Mon Apr 25 19:18:04 2022 +0800

HBASE-26943 HMaster page style display confusion (#4369)

Signed-off-by: Yu Li 
Signed-off-by: Nick Dimiduk 
---
 .../src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java  | 2 +-
 .../jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon  | 2 +-
 hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp | 4 ++--
 .../src/main/resources/hbase-webapps/master/operationDetails.jsp  | 2 +-
 hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp   | 2 +-
 .../src/main/resources/hbase-webapps/master/processMaster.jsp | 2 +-
 .../src/main/resources/hbase-webapps/master/tablesDetailed.jsp| 2 +-
 hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css| 2 ++
 8 files changed, 10 insertions(+), 8 deletions(-)

diff --git 
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java 
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 611316d9ec6..baf970b9aff 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -396,7 +396,7 @@ public final class LogLevel {
 }
 
 static final String FORMS = "\n"
-+ "\n" + "\n"
++ "\n" + "\n"
 + "Get/Set Log Level\n" + "\n" + "\n" + 
"Actions:" + ""
 + "\n" + "\n" + "\n"
 + "\n" + "\n"
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 5582532a95c..aff4e537495 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -174,7 +174,7 @@ AssignmentManager assignmentManager = 
master.getAssignmentManager();
 
 
<%if master.isActiveMaster() %>
-
+
 
 Master <% master.getServerName().getHostname() 
%>
 
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
index 69b95e1a118..29fa5627838 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
@@ -83,7 +83,7 @@
 
 
   <% if (!master.isInitialized()) { %>
-  
+  
 
   Master is not initialized
 
@@ -91,7 +91,7 @@
   
   <% } else { %>
 
-  
+  
 
   This page displays two reports: the HBCK Chore Report 
and
 the CatalogJanitor Consistency Issues report. Only report 
titles
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
index e6049558b28..6d4f9eece2d 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
@@ -49,7 +49,7 @@
 
 
 
-  
+  
 
 Operations Details
 HBase uses some fixed-size ring buffers to maintain rolling window 
history of specific server-side operation details.
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
index fba9a42b94e..0ac2ff3908d 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
@@ -68,7 +68,7 @@
 
 
 
-  
+  
 
   Procedure Time Statistics
 
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
index 39ac83c337d..65bc10a65f1 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
@@ -49,7 +49,7 @@ pageContext.setAttribute("pageTitle", "Process info for PID: 
" + JSONMetricUtil.
 
 
 
-  
+  
   
   <%= JSONMetricUtil.getCommmand().split(" ")[0] %>
   
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master

[hbase] 02/02: HBASE-26943 HMaster page style display confusion (#4369)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit 7d5bf1c4f275b197b260980b255d044a61ef44ec
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Mon Apr 25 19:18:04 2022 +0800

HBASE-26943 HMaster page style display confusion (#4369)

Signed-off-by: Yu Li 
Signed-off-by: Nick Dimiduk 
---
 .../src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java  | 2 +-
 .../jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon  | 2 +-
 hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp | 4 ++--
 .../src/main/resources/hbase-webapps/master/operationDetails.jsp  | 2 +-
 hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp   | 2 +-
 .../src/main/resources/hbase-webapps/master/processMaster.jsp | 2 +-
 .../src/main/resources/hbase-webapps/master/tablesDetailed.jsp| 2 +-
 hbase-server/src/main/resources/hbase-webapps/static/css/hbase.css| 2 ++
 8 files changed, 10 insertions(+), 8 deletions(-)

diff --git 
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java 
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 611316d9ec6..baf970b9aff 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -396,7 +396,7 @@ public final class LogLevel {
 }
 
 static final String FORMS = "\n"
-+ "\n" + "\n"
++ "\n" + "\n"
 + "Get/Set Log Level\n" + "\n" + "\n" + 
"Actions:" + ""
 + "\n" + "\n" + "\n"
 + "\n" + "\n"
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 65c9e0b66c6..c55013dd67b 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -174,7 +174,7 @@ AssignmentManager assignmentManager = 
master.getAssignmentManager();
 
 
<%if master.isActiveMaster() %>
-
+
 
 Master <% master.getServerName().getHostname() 
%>
 
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
index 69b95e1a118..29fa5627838 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp
@@ -83,7 +83,7 @@
 
 
   <% if (!master.isInitialized()) { %>
-  
+  
 
   Master is not initialized
 
@@ -91,7 +91,7 @@
   
   <% } else { %>
 
-  
+  
 
   This page displays two reports: the HBCK Chore Report 
and
 the CatalogJanitor Consistency Issues report. Only report 
titles
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
index e6049558b28..6d4f9eece2d 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/operationDetails.jsp
@@ -49,7 +49,7 @@
 
 
 
-  
+  
 
 Operations Details
 HBase uses some fixed-size ring buffers to maintain rolling window 
history of specific server-side operation details.
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
index fba9a42b94e..0ac2ff3908d 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/procedures.jsp
@@ -68,7 +68,7 @@
 
 
 
-  
+  
 
   Procedure Time Statistics
 
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
index 39ac83c337d..65bc10a65f1 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/processMaster.jsp
@@ -49,7 +49,7 @@ pageContext.setAttribute("pageTitle", "Process info for PID: 
" + JSONMetricUtil.
 
 
 
-  
+  
   
   <%= JSONMetricUtil.getCommmand().split(" ")[0] %>
   
diff --git 
a/hbase-server/src/main/resources/hbase-webapps/master/tablesDetailed.jsp 
b/hbase-server/src/main/resources/hbase-webapps/master/tablesDetailed.jsp
index 2ad44a3d68b..7e0d623e686 100644
--- a/hbase-server/src/main/resources/hbase-webapps/m

[hbase] branch master updated (f4eb9071fdc -> 7d5bf1c4f27)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from f4eb9071fdc HBASE-26917 Do not add --threads when running 'mvn site' 
(#4354)
 new 23971b2861a Revert "HBASE-26943 HMaster page style display confusion" 
and "HBASE-26943 Formatting bootstrap.min.css" (#4369)
 new 7d5bf1c4f27 HBASE-26943 HMaster page style display confusion (#4369)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/hadoop/hbase/http/log/LogLevel.java |2 +-
 .../hbase/tmpl/master/MasterStatusTmpl.jamon   |2 +-
 .../main/resources/hbase-webapps/master/hbck.jsp   |4 +-
 .../hbase-webapps/master/operationDetails.jsp  |2 +-
 .../resources/hbase-webapps/master/procedures.jsp  |2 +-
 .../hbase-webapps/master/processMaster.jsp |2 +-
 .../hbase-webapps/master/tablesDetailed.jsp|2 +-
 .../hbase-webapps/static/css/bootstrap.min.css | 8028 +---
 .../resources/hbase-webapps/static/css/hbase.css   |2 +
 9 files changed, 12 insertions(+), 8034 deletions(-)

[hbase] branch master updated: HBASE-26951 HMaster should exit gracefully, when stopped via hbase-daemon.sh (#4358)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new ba713ac379a HBASE-26951 HMaster should exit gracefully, when stopped 
via hbase-daemon.sh (#4358)
ba713ac379a is described below

commit ba713ac379a9d2faa2d875eebd9315e7b9485879
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Wed Apr 20 17:37:18 2022 +0800

HBASE-26951 HMaster should exit gracefully, when stopped via 
hbase-daemon.sh (#4358)

Signed-off-by: Yu Li 
Signed-off-by: Duo Zhang 
---
 .../org/apache/hadoop/hbase/HBaseServerBase.java | 20 
 .../java/org/apache/hadoop/hbase/master/HMaster.java |  1 +
 .../hadoop/hbase/regionserver/HRegionServer.java |  2 +-
 .../org/apache/hadoop/hbase/master/TestMaster.java   |  7 ++-
 .../hbase/regionserver/TestRegionServerNoMaster.java |  7 +++
 5 files changed, 35 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
index c28ea29215a..91170d41531 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.namequeues.NamedQueueRecorder;
 import org.apache.hadoop.hbase.regionserver.ChunkCreator;
 import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
 import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
+import org.apache.hadoop.hbase.regionserver.ShutdownHook;
 import org.apache.hadoop.hbase.security.Superusers;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.UserProvider;
@@ -91,6 +92,9 @@ public abstract class HBaseServerBase> extends
   // of HRegionServer in isolation.
   protected volatile boolean stopped = false;
 
+  // Only for testing
+  private boolean isShutdownHookInstalled = false;
+
   /**
* This servers startcode.
*/
@@ -447,6 +451,22 @@ public abstract class HBaseServerBase> extends
 }
   }
 
+  /**
+   * In order to register ShutdownHook, this method is called
+   * when HMaster and HRegionServer are started.
+   * For details, please refer to HBASE-26951
+   */
+  protected final void installShutdownHook() {
+ShutdownHook.install(conf, dataFs, this, Thread.currentThread());
+isShutdownHookInstalled = true;
+  }
+
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+allowedOnPath = ".*/src/test/.*")
+  public boolean isShutdownHookInstalled() {
+return isShutdownHookInstalled;
+  }
+
   @Override
   public ServerName getServerName() {
 return serverName;
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 01e4e5cd961..91e18a6f88f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -553,6 +553,7 @@ public class HMaster extends 
HBaseServerBase implements Maste
   @Override
   public void run() {
 try {
+  installShutdownHook();
   registerConfigurationObservers();
   Threads.setDaemonThreadRunning(new Thread(() -> {
 try {
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c56157e6a0e..3aadf7a9e54 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -760,7 +760,7 @@ public class HRegionServer extends 
HBaseServerBase
 
 try {
   if (!isStopped() && !isAborted()) {
-ShutdownHook.install(conf, dataFs, this, Thread.currentThread());
+installShutdownHook();
 // Initialize the RegionServerCoprocessorHost now that our ephemeral
 // node was created, in case any coprocessors want to use ZooKeeper
 this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 65a205fd3c9..8dcc36bbc0d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -305,5 +305,10 @@ public class TestMaster {
 // Assert lock gets put in place again.
 assertTrue(fs.exists(hbckLockPath));
   }
-}
 
+  @Test
+  public void testInstallShutdownHook() throws IOException {
+// Test for H

[hbase] branch master updated (67ac16c8da -> 242a194b3c)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from 67ac16c8da HBASE-26943 HMaster page style display confusion
 add 463a92da65 Revert "HBASE-26943 HMaster page style display confusion" 
to correct author information
 add 58d40dbcb5 Revert "HBASE-26943 Formatting bootstrap.min.css" to 
correct author information
 add d0318732b7 HBASE-26943 Formatting bootstrap.min.css
 add 242a194b3c HBASE-26943 HMaster page style display confusion

No new revisions were added by this update.

Summary of changes:

[hbase] branch master updated (b7ffb56927 -> 67ac16c8da)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


from b7ffb56927 Revert "HBASE-26941 LocalHBaseCluster.waitOnRegionServer 
should quit while thread is interrupted (#4333)"
 new c5158aef93 HBASE-26943 Formatting bootstrap.min.css
 new 67ac16c8da HBASE-26943 HMaster page style display confusion

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../hbase-webapps/static/css/bootstrap.min.css | 8028 +++-
 1 file changed, 8026 insertions(+), 2 deletions(-)

[hbase] 02/02: HBASE-26943 HMaster page style display confusion

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit 67ac16c8da731a1bd1cc040c8898e4b66b23e983
Author: liangjunhe.hlj 
AuthorDate: Tue Apr 12 23:18:08 2022 +0800

HBASE-26943 HMaster page style display confusion

Signed-off-by: Yu Li 
---
 .../src/main/resources/hbase-webapps/static/css/bootstrap.min.css  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/resources/hbase-webapps/static/css/bootstrap.min.css 
b/hbase-server/src/main/resources/hbase-webapps/static/css/bootstrap.min.css
index 49fdc4c1a9..7b0c38da3c 100644
--- a/hbase-server/src/main/resources/hbase-webapps/static/css/bootstrap.min.css
+++ b/hbase-server/src/main/resources/hbase-webapps/static/css/bootstrap.min.css
@@ -2016,7 +2016,8 @@ pre code {
 
 .row {
   margin-right: -15px;
-  margin-left: -15px
+  margin-left: -15px;
+  margin-top: 20px
 }
 
 .row-no-gutters {

[hbase] branch branch-1 updated: HBASE-21777 Only log compaction pressure throughput when changed or traced (addendum)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-1 by this push:
 new e4ab715  HBASE-21777 Only log compaction pressure throughput when 
changed or traced (addendum)
e4ab715 is described below

commit e4ab71590142c8481fbedf15450504a5f698cbab
Author: Yu Li 
AuthorDate: Thu May 9 00:59:10 2019 +0800

HBASE-21777 Only log compaction pressure throughput when changed or traced 
(addendum)

Fix the findbugs warning
---
 .../throttle/PressureAwareCompactionThroughputController.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index 7df07bb..2dc5817 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -100,7 +100,7 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   * compactionPressure;
 }
 if (LOG.isDebugEnabled()) {
-  if (maxThroughputToSet != getMaxThroughput()) {
+  if (Math.abs(maxThroughputToSet - getMaxThroughput()) < .001) {
 LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
 + throughputDesc(maxThroughputToSet));
   } else if (LOG.isTraceEnabled()) {

[hbase] branch branch-2 updated: HBASE-21777 Only log compaction pressure throughput when changed or traced (addendum)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 07c71d6  HBASE-21777 Only log compaction pressure throughput when 
changed or traced (addendum)
07c71d6 is described below

commit 07c71d630cd293ab55ac85e9bfa06033598134c4
Author: Yu Li 
AuthorDate: Thu May 9 00:59:10 2019 +0800

HBASE-21777 Only log compaction pressure throughput when changed or traced 
(addendum)

Fix the findbugs warning
---
 .../throttle/PressureAwareCompactionThroughputController.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index 63aa646..740152d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -100,7 +100,7 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   * compactionPressure;
 }
 if (LOG.isDebugEnabled()) {
-  if (maxThroughputToSet != getMaxThroughput()) {
+  if (Math.abs(maxThroughputToSet - getMaxThroughput()) < .001) {
 LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
 + throughputDesc(maxThroughputToSet));
   } else if (LOG.isTraceEnabled()) {

[hbase] branch master updated: HBASE-21777 Only log compaction pressure throughput when changed or traced (addendum)

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new 12bf75b  HBASE-21777 Only log compaction pressure throughput when 
changed or traced (addendum)
12bf75b is described below

commit 12bf75bb2aac8d519f607c50d4831a6bd9799875
Author: Yu Li 
AuthorDate: Thu May 9 00:59:10 2019 +0800

HBASE-21777 Only log compaction pressure throughput when changed or traced 
(addendum)

Fix the findbugs warning
---
 .../throttle/PressureAwareCompactionThroughputController.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index 63aa646..740152d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -100,7 +100,7 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   * compactionPressure;
 }
 if (LOG.isDebugEnabled()) {
-  if (maxThroughputToSet != getMaxThroughput()) {
+  if (Math.abs(maxThroughputToSet - getMaxThroughput()) < .001) {
 LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
 + throughputDesc(maxThroughputToSet));
   } else if (LOG.isTraceEnabled()) {

[hbase] branch branch-2 updated: HBASE-21777 Only log compaction pressure throughput when changed or traced

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new a591c60  HBASE-21777 Only log compaction pressure throughput when 
changed or traced
a591c60 is described below

commit a591c60368912b8c595bed4bfcac48fcc983cb70
Author: Tak Lon (Stephen) Wu 
AuthorDate: Tue May 7 11:28:08 2019 -0700

HBASE-21777 Only log compaction pressure throughput when changed or traced

Signed-off-by: Yu Li 
Signed-off-by: Andrew Purtell 
---
 .../PressureAwareCompactionThroughputController.java| 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index b3c7bf3..63aa646 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -99,11 +99,14 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   maxThroughputLowerBound + (maxThroughputUpperBound - 
maxThroughputLowerBound)
   * compactionPressure;
 }
-if (LOG.isTraceEnabled()) {
-  // TODO: FIX!!! Don't log unless some activity or a change in config. 
Making TRACE
-  // in the meantime.
-  LOG.trace("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
-  + throughputDesc(maxThroughputToSet));
+if (LOG.isDebugEnabled()) {
+  if (maxThroughputToSet != getMaxThroughput()) {
+LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
++ throughputDesc(maxThroughputToSet));
+  } else if (LOG.isTraceEnabled()) {
+LOG.trace("CompactionPressure is " + compactionPressure + ", keep 
throughput throttling to "
++ throughputDesc(maxThroughputToSet));
+  }
 }
 this.setMaxThroughput(maxThroughputToSet);
   }

[hbase] branch master updated: HBASE-21777 Only log compaction pressure throughput when changed or traced

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new b5b89f7  HBASE-21777 Only log compaction pressure throughput when 
changed or traced
b5b89f7 is described below

commit b5b89f7c742c3c60c307c49866f207e2d219e3a3
Author: Tak Lon (Stephen) Wu 
AuthorDate: Tue May 7 11:28:08 2019 -0700

HBASE-21777 Only log compaction pressure throughput when changed or traced

Signed-off-by: Yu Li 
Signed-off-by: Andrew Purtell 
---
 .../PressureAwareCompactionThroughputController.java| 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index b3c7bf3..63aa646 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -99,11 +99,14 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   maxThroughputLowerBound + (maxThroughputUpperBound - 
maxThroughputLowerBound)
   * compactionPressure;
 }
-if (LOG.isTraceEnabled()) {
-  // TODO: FIX!!! Don't log unless some activity or a change in config. 
Making TRACE
-  // in the meantime.
-  LOG.trace("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
-  + throughputDesc(maxThroughputToSet));
+if (LOG.isDebugEnabled()) {
+  if (maxThroughputToSet != getMaxThroughput()) {
+LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
++ throughputDesc(maxThroughputToSet));
+  } else if (LOG.isTraceEnabled()) {
+LOG.trace("CompactionPressure is " + compactionPressure + ", keep 
throughput throttling to "
++ throughputDesc(maxThroughputToSet));
+  }
 }
 this.setMaxThroughput(maxThroughputToSet);
   }

[hbase] branch branch-1 updated: HBASE-21777 Only log compaction pressure throughput when changed or traced

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-1 by this push:
 new fbd53bf  HBASE-21777 Only log compaction pressure throughput when 
changed or traced
fbd53bf is described below

commit fbd53bfefb08e65d4118009baa85b4bde43e731d
Author: Tak Lon (Stephen) Wu 
AuthorDate: Tue May 7 11:28:08 2019 -0700

HBASE-21777 Only log compaction pressure throughput when changed or traced
---
 .../throttle/PressureAwareCompactionThroughputController.java| 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
index c0d3b74..7df07bb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/PressureAwareCompactionThroughputController.java
@@ -100,8 +100,13 @@ public class PressureAwareCompactionThroughputController 
extends PressureAwareTh
   * compactionPressure;
 }
 if (LOG.isDebugEnabled()) {
-  LOG.debug("compactionPressure is " + compactionPressure + ", tune 
compaction throughput to "
-  + throughputDesc(maxThroughputToSet));
+  if (maxThroughputToSet != getMaxThroughput()) {
+LOG.debug("CompactionPressure is " + compactionPressure + ", tune 
throughput to "
++ throughputDesc(maxThroughputToSet));
+  } else if (LOG.isTraceEnabled()) {
+LOG.trace("CompactionPressure is " + compactionPressure + ", keep 
throughput throttling to "
++ throughputDesc(maxThroughputToSet));
+  }
 }
 this.setMaxThroughput(maxThroughputToSet);
   }

[hbase] branch branch-2 updated: HBASE-22283 Print row and table information when failed to get region location

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
 new 54b944a  HBASE-22283 Print row and table information when failed to 
get region location
54b944a is described below

commit 54b944a10f012ea0649482af6063d78e7a3c12d6
Author: Yu Li 
AuthorDate: Wed Apr 24 15:44:13 2019 +0800

HBASE-22283 Print row and table information when failed to get region 
location
---
 .../client/RpcRetryingCallerWithReadReplicas.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
index a0be0bf..260cb8c 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hbase.client;
 
 
+import static org.apache.hadoop.hbase.HConstants.PRIORITY_UNSET;
+
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.Collections;
@@ -34,17 +36,17 @@ import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-import static org.apache.hadoop.hbase.HConstants.PRIORITY_UNSET;
 
 /**
  * Caller that goes to replica if the primary region does no answer within a 
configurable
@@ -330,10 +332,12 @@ public class RpcRetryingCallerWithReadReplicas {
 } catch (DoNotRetryIOException | InterruptedIOException | 
RetriesExhaustedException e) {
   throw e;
 } catch (IOException e) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId, e);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName, e);
 }
 if (rl == null) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName);
 }
 
 return rl;

[hbase] branch branch-1 updated: HBASE-22283 Print row and table information when failed to get region location

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-1 by this push:
 new 4648ab1  HBASE-22283 Print row and table information when failed to 
get region location
4648ab1 is described below

commit 4648ab1db653557852d4bdcd49ecb6f84f49be70
Author: Yu Li 
AuthorDate: Wed Apr 24 15:44:13 2019 +0800

HBASE-22283 Print row and table information when failed to get region 
location
---
 .../hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
index 5a41233..75bb649 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 /**
@@ -366,10 +367,12 @@ public class RpcRetryingCallerWithReadReplicas {
 } catch (InterruptedIOException e) {
   throw e;
 } catch (IOException e) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId, e);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName, e);
 }
 if (rl == null) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName);
 }
 
 return rl;

[hbase] branch master updated: HBASE-22283 Print row and table information when failed to get region location

[liyu]

This is an automated email from the ASF dual-hosted git repository.

liyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
 new ab3d6cf  HBASE-22283 Print row and table information when failed to 
get region location
ab3d6cf is described below

commit ab3d6cf811ea1c40e570551281e91571a038f2d1
Author: Yu Li 
AuthorDate: Wed Apr 24 15:44:13 2019 +0800

HBASE-22283 Print row and table information when failed to get region 
location
---
 .../client/RpcRetryingCallerWithReadReplicas.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
index 4a31cff..c82fcb8 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hbase.client;
 
 
+import static org.apache.hadoop.hbase.HConstants.PRIORITY_UNSET;
+
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.Collections;
@@ -34,17 +36,17 @@ import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-import static org.apache.hadoop.hbase.HConstants.PRIORITY_UNSET;
 
 /**
  * Caller that goes to replica if the primary region does no answer within a 
configurable
@@ -329,10 +331,12 @@ public class RpcRetryingCallerWithReadReplicas {
 } catch (DoNotRetryIOException | InterruptedIOException | 
RetriesExhaustedException e) {
   throw e;
 } catch (IOException e) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId, e);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName, e);
 }
 if (rl == null) {
-  throw new RetriesExhaustedException("Can't get the location for replica 
" + replicaId);
+  throw new RetriesExhaustedException("Cannot get the location for 
replica" + replicaId
+  + " of region for " + Bytes.toStringBinary(row) + " in " + 
tableName);
 }
 
 return rl;

hbase git commit: HBASE-20559 Backport HBASE-18083 (Make large/small file clean thread number configurable in HFileCleaner) to branch-1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.4 4e30572e7 -> 6d6b5b777


HBASE-20559 Backport HBASE-18083 (Make large/small file clean thread number 
configurable in HFileCleaner) to branch-1

The last port commit of HBASE-20555

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d6b5b77
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d6b5b77
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d6b5b77

Branch: refs/heads/branch-1.4
Commit: 6d6b5b7773bbefe6d8136aea4f2bc5e3b8e673a4
Parents: 4e30572
Author: TAK LON WU 
Authored: Mon Jul 23 11:23:57 2018 +0800
Committer: Yu Li 
Committed: Mon Jul 23 11:36:09 2018 +0800

--
 .../hbase/master/cleaner/HFileCleaner.java  | 154 +--
 .../hbase/master/cleaner/TestHFileCleaner.java  |  13 +-
 2 files changed, 120 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6d6b5b77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 70548b4..8f0b4be 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -66,6 +67,16 @@ public class HFileCleaner extends 
CleanerChore impleme
   "hbase.regionserver.hfilecleaner.small.queue.size";
   public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
 
+  // Configuration key for large file delete thread number
+  public final static String LARGE_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.large.thread.count";
+  public final static int DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER = 1;
+
+  // Configuration key for small file delete thread number
+  public final static String SMALL_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.small.thread.count";
+  public final static int DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER = 1;
+
   private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
 
   StealJobQueue largeFileQueue;
@@ -73,11 +84,13 @@ public class HFileCleaner extends 
CleanerChore impleme
   private int throttlePoint;
   private int largeQueueInitSize;
   private int smallQueueInitSize;
+  private int largeFileDeleteThreadNumber;
+  private int smallFileDeleteThreadNumber;
   private List threads = new ArrayList();
   private boolean running;
 
-  private long deletedLargeFiles = 0L;
-  private long deletedSmallFiles = 0L;
+  private AtomicLong deletedLargeFiles = new AtomicLong();
+  private AtomicLong deletedSmallFiles = new AtomicLong();
 
   /**
* @param period the period of time to sleep between each run
@@ -99,6 +112,10 @@ public class HFileCleaner extends 
CleanerChore impleme
 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
 largeFileQueue = new StealJobQueue<>(largeQueueInitSize, 
smallQueueInitSize);
 smallFileQueue = largeFileQueue.getStealFromQueue();
+largeFileDeleteThreadNumber =
+conf.getInt(LARGE_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER);
+smallFileDeleteThreadNumber =
+conf.getInt(SMALL_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER);
 startHFileDeleteThreads();
   }
 
@@ -182,30 +199,34 @@ public class HFileCleaner extends 
CleanerChore impleme
 final String n = Thread.currentThread().getName();
 running = true;
 // start thread for large file deletion
-Thread large = new Thread() {
-  @Override
-  public void run() {
-consumerLoop(largeFileQueue);
-  }
-};
-large.setDaemon(true);
-large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
-large.start();
-LOG.debug("Starting hfile cleaner for large files: " + large.getName());
-threads.add(large);
+for (int i = 0; i < largeFileDeleteThreadNumber; i++) {
+  Thread large = new Thread() {
+@Override
+public void run() {
+  consumerLoop(largeFileQueue);
+}
+  };
+  large.setDaemon(true);
+  large.setName(n + "-HFileCleaner.large." + i + "-" + 
System.currentTimeMillis());
+  large.start();
+  LOG.debug("St

hbase git commit: HBASE-20559 Backport HBASE-18083 (Make large/small file clean thread number configurable in HFileCleaner) to branch-1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 780724b15 -> 896b69a0f


HBASE-20559 Backport HBASE-18083 (Make large/small file clean thread number 
configurable in HFileCleaner) to branch-1

The last port commit of HBASE-20555

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/896b69a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/896b69a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/896b69a0

Branch: refs/heads/branch-1
Commit: 896b69a0fc6c3164d33a1269e0d7ac87d89b3d90
Parents: 780724b
Author: TAK LON WU 
Authored: Mon Jul 23 11:23:57 2018 +0800
Committer: Yu Li 
Committed: Mon Jul 23 11:36:47 2018 +0800

--
 .../hbase/master/cleaner/HFileCleaner.java  | 154 +--
 .../hbase/master/cleaner/TestHFileCleaner.java  |  13 +-
 2 files changed, 120 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/896b69a0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 70548b4..8f0b4be 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -66,6 +67,16 @@ public class HFileCleaner extends 
CleanerChore impleme
   "hbase.regionserver.hfilecleaner.small.queue.size";
   public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
 
+  // Configuration key for large file delete thread number
+  public final static String LARGE_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.large.thread.count";
+  public final static int DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER = 1;
+
+  // Configuration key for small file delete thread number
+  public final static String SMALL_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.small.thread.count";
+  public final static int DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER = 1;
+
   private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
 
   StealJobQueue largeFileQueue;
@@ -73,11 +84,13 @@ public class HFileCleaner extends 
CleanerChore impleme
   private int throttlePoint;
   private int largeQueueInitSize;
   private int smallQueueInitSize;
+  private int largeFileDeleteThreadNumber;
+  private int smallFileDeleteThreadNumber;
   private List threads = new ArrayList();
   private boolean running;
 
-  private long deletedLargeFiles = 0L;
-  private long deletedSmallFiles = 0L;
+  private AtomicLong deletedLargeFiles = new AtomicLong();
+  private AtomicLong deletedSmallFiles = new AtomicLong();
 
   /**
* @param period the period of time to sleep between each run
@@ -99,6 +112,10 @@ public class HFileCleaner extends 
CleanerChore impleme
 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
 largeFileQueue = new StealJobQueue<>(largeQueueInitSize, 
smallQueueInitSize);
 smallFileQueue = largeFileQueue.getStealFromQueue();
+largeFileDeleteThreadNumber =
+conf.getInt(LARGE_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER);
+smallFileDeleteThreadNumber =
+conf.getInt(SMALL_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER);
 startHFileDeleteThreads();
   }
 
@@ -182,30 +199,34 @@ public class HFileCleaner extends 
CleanerChore impleme
 final String n = Thread.currentThread().getName();
 running = true;
 // start thread for large file deletion
-Thread large = new Thread() {
-  @Override
-  public void run() {
-consumerLoop(largeFileQueue);
-  }
-};
-large.setDaemon(true);
-large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
-large.start();
-LOG.debug("Starting hfile cleaner for large files: " + large.getName());
-threads.add(large);
+for (int i = 0; i < largeFileDeleteThreadNumber; i++) {
+  Thread large = new Thread() {
+@Override
+public void run() {
+  consumerLoop(largeFileQueue);
+}
+  };
+  large.setDaemon(true);
+  large.setName(n + "-HFileCleaner.large." + i + "-" + 
System.currentTimeMillis());
+  large.start();
+  LOG.debug("Starti

hbase git commit: HBASE-20907 Fix Intermittent failure on TestProcedurePriority

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2.0 9c44f9594 -> f17c97bc1


HBASE-20907 Fix Intermittent failure on TestProcedurePriority


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f17c97bc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f17c97bc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f17c97bc

Branch: refs/heads/branch-2.0
Commit: f17c97bc17a57c7226ff8eecbaaf54d1c540c08c
Parents: 9c44f95
Author: Yu Li 
Authored: Wed Jul 18 16:02:59 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 19 12:03:03 2018 +0800

--
 .../hadoop/hbase/master/procedure/TestProcedurePriority.java   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f17c97bc/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
index 9f5741c..7386b2f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
@@ -136,6 +136,8 @@ public class TestProcedurePriority {
   .stream().filter(t -> 
!t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty())
   .findAny().get();
 HRegionServer rsNoMeta = 
UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer());
+// wait for NS table initialization to avoid our error inject affecting 
master initialization
+UTIL.waitTableAvailable(TableName.NAMESPACE_TABLE_NAME);
 FAIL = true;
 UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName());
 // wait until all the worker thread are stuck, which means that the stuck 
checker will start to

hbase git commit: HBASE-20907 Fix Intermittent failure on TestProcedurePriority

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2.1 cecce16fa -> 9ac26b80b


HBASE-20907 Fix Intermittent failure on TestProcedurePriority


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9ac26b80
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9ac26b80
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9ac26b80

Branch: refs/heads/branch-2.1
Commit: 9ac26b80b2d2954e963685c2e5ccd4d8aa3e
Parents: cecce16
Author: Yu Li 
Authored: Wed Jul 18 16:02:59 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 19 12:01:29 2018 +0800

--
 .../hadoop/hbase/master/procedure/TestProcedurePriority.java   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9ac26b80/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
index 9f5741c..7386b2f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
@@ -136,6 +136,8 @@ public class TestProcedurePriority {
   .stream().filter(t -> 
!t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty())
   .findAny().get();
 HRegionServer rsNoMeta = 
UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer());
+// wait for NS table initialization to avoid our error inject affecting 
master initialization
+UTIL.waitTableAvailable(TableName.NAMESPACE_TABLE_NAME);
 FAIL = true;
 UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName());
 // wait until all the worker thread are stuck, which means that the stuck 
checker will start to

hbase git commit: HBASE-20907 Fix Intermittent failure on TestProcedurePriority

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 532d84d6d -> e464230bd


HBASE-20907 Fix Intermittent failure on TestProcedurePriority


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e464230b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e464230b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e464230b

Branch: refs/heads/branch-2
Commit: e464230bde66fa3f85e3ac7afed0437dca3bfe2e
Parents: 532d84d
Author: Yu Li 
Authored: Wed Jul 18 16:02:59 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 19 12:01:16 2018 +0800

--
 .../hadoop/hbase/master/procedure/TestProcedurePriority.java   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e464230b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
index 9f5741c..7386b2f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
@@ -136,6 +136,8 @@ public class TestProcedurePriority {
   .stream().filter(t -> 
!t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty())
   .findAny().get();
 HRegionServer rsNoMeta = 
UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer());
+// wait for NS table initialization to avoid our error inject affecting 
master initialization
+UTIL.waitTableAvailable(TableName.NAMESPACE_TABLE_NAME);
 FAIL = true;
 UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName());
 // wait until all the worker thread are stuck, which means that the stuck 
checker will start to

hbase git commit: HBASE-20907 Fix Intermittent failure on TestProcedurePriority

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 2bf5e46a3 -> 619e0fdab


HBASE-20907 Fix Intermittent failure on TestProcedurePriority


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/619e0fda
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/619e0fda
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/619e0fda

Branch: refs/heads/master
Commit: 619e0fdab88ab0599ef157b430e85ac0752eb6c7
Parents: 2bf5e46
Author: Yu Li 
Authored: Wed Jul 18 16:02:59 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 19 12:00:55 2018 +0800

--
 .../hadoop/hbase/master/procedure/TestProcedurePriority.java   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/619e0fda/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
index 9f5741c..7386b2f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java
@@ -136,6 +136,8 @@ public class TestProcedurePriority {
   .stream().filter(t -> 
!t.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty())
   .findAny().get();
 HRegionServer rsNoMeta = 
UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer());
+// wait for NS table initialization to avoid our error inject affecting 
master initialization
+UTIL.waitTableAvailable(TableName.NAMESPACE_TABLE_NAME);
 FAIL = true;
 UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName());
 // wait until all the worker thread are stuck, which means that the stuck 
checker will start to

hbase git commit: HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2.1 368b1b106 -> da39c6d39


HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da39c6d3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da39c6d3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da39c6d3

Branch: refs/heads/branch-2.1
Commit: da39c6d397655232de68286705afa047f3b06e80
Parents: 368b1b1
Author: Yu Li 
Authored: Thu Jul 12 10:14:36 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 12 14:08:25 2018 +0800

--
 dev-support/hbase-personality.sh| 12 
 .../org/apache/hadoop/hbase/util/CommonFSUtils.java |  4 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/da39c6d3/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index da96019..6d1b87f 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -179,6 +179,18 @@ function personality_modules
 if [ -n "${BUILD_ID}" ]; then
   extra="${extra} -Dbuild.id=${BUILD_ID}"
 fi
+
+# If the set of changed files includes CommonFSUtils then add the 
hbase-server
+# module to the set of modules (if not already included) to be tested
+for f in "${CHANGED_FILES[@]}"
+do
+  if [[ "${f}" =~ CommonFSUtils ]]; then
+if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ 
\. ]]; then
+  MODULES+=("hbase-server")
+fi
+break
+  fi
+done
   fi
 
   for module in "${MODULES[@]}"; do

http://git-wip-us.apache.org/repos/asf/hbase/blob/da39c6d3/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 76a3601..2e480e6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -51,6 +51,10 @@ import 
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 
 /**
  * Utility methods for interacting with the underlying file system.
+ * 
+ * Note that {@link #setStoragePolicy(FileSystem, Path, String)} is tested in 
TestFSUtils and
+ * pre-commit will run the hbase-server tests if there's code change in this 
class. See
+ * https://issues.apache.org/jira/browse/HBASE-20838";>HBASE-20838 
for more details.
  */
 @InterfaceAudience.Private
 public abstract class CommonFSUtils {

hbase git commit: HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 59830b007 -> 492c0fcd2


HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/492c0fcd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/492c0fcd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/492c0fcd

Branch: refs/heads/branch-2
Commit: 492c0fcd2e4380a21928985b274d4818f80acb41
Parents: 59830b0
Author: Yu Li 
Authored: Thu Jul 12 10:14:36 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 12 10:17:38 2018 +0800

--
 dev-support/hbase-personality.sh| 12 
 .../org/apache/hadoop/hbase/util/CommonFSUtils.java |  4 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/492c0fcd/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index da96019..6d1b87f 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -179,6 +179,18 @@ function personality_modules
 if [ -n "${BUILD_ID}" ]; then
   extra="${extra} -Dbuild.id=${BUILD_ID}"
 fi
+
+# If the set of changed files includes CommonFSUtils then add the 
hbase-server
+# module to the set of modules (if not already included) to be tested
+for f in "${CHANGED_FILES[@]}"
+do
+  if [[ "${f}" =~ CommonFSUtils ]]; then
+if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ 
\. ]]; then
+  MODULES+=("hbase-server")
+fi
+break
+  fi
+done
   fi
 
   for module in "${MODULES[@]}"; do

http://git-wip-us.apache.org/repos/asf/hbase/blob/492c0fcd/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 76a3601..2e480e6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -51,6 +51,10 @@ import 
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 
 /**
  * Utility methods for interacting with the underlying file system.
+ * 
+ * Note that {@link #setStoragePolicy(FileSystem, Path, String)} is tested in 
TestFSUtils and
+ * pre-commit will run the hbase-server tests if there's code change in this 
class. See
+ * https://issues.apache.org/jira/browse/HBASE-20838";>HBASE-20838 
for more details.
  */
 @InterfaceAudience.Private
 public abstract class CommonFSUtils {

hbase git commit: HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 51527f551 -> c55acb0b0


HBASE-20838 Include hbase-server in precommit test if CommonFSUtils is changed


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c55acb0b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c55acb0b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c55acb0b

Branch: refs/heads/master
Commit: c55acb0b0363525f8d3ff52cbcbd05f3bbdb20a4
Parents: 51527f5
Author: Yu Li 
Authored: Thu Jul 12 10:14:36 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 12 10:16:11 2018 +0800

--
 dev-support/hbase-personality.sh| 12 
 .../org/apache/hadoop/hbase/util/CommonFSUtils.java |  4 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c55acb0b/dev-support/hbase-personality.sh
--
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 8004167..1ca32ee 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -180,6 +180,18 @@ function personality_modules
 if [ -n "${BUILD_ID}" ]; then
   extra="${extra} -Dbuild.id=${BUILD_ID}"
 fi
+
+# If the set of changed files includes CommonFSUtils then add the 
hbase-server
+# module to the set of modules (if not already included) to be tested
+for f in "${CHANGED_FILES[@]}"
+do
+  if [[ "${f}" =~ CommonFSUtils ]]; then
+if [[ ! "${MODULES[*]}" =~ hbase-server ]] && [[ ! "${MODULES[*]}" =~ 
\. ]]; then
+  MODULES+=("hbase-server")
+fi
+break
+  fi
+done
   fi
 
   for module in "${MODULES[@]}"; do

http://git-wip-us.apache.org/repos/asf/hbase/blob/c55acb0b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 76a3601..2e480e6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -51,6 +51,10 @@ import 
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 
 /**
  * Utility methods for interacting with the underlying file system.
+ * 
+ * Note that {@link #setStoragePolicy(FileSystem, Path, String)} is tested in 
TestFSUtils and
+ * pre-commit will run the hbase-server tests if there's code change in this 
class. See
+ * https://issues.apache.org/jira/browse/HBASE-20838";>HBASE-20838 
for more details.
  */
 @InterfaceAudience.Private
 public abstract class CommonFSUtils {

hbase git commit: HBASE-20691 Change the default WAL storage policy back to "NONE""

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 a25cf7bbd -> 50f7894cc


HBASE-20691 Change the default WAL storage policy back to "NONE""

This partially reverts HBASE-19858 and adds more doc about why we choose "NONE" 
as the default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50f7894c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50f7894c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50f7894c

Branch: refs/heads/branch-1
Commit: 50f7894cc488bd5d5f40284a9a95d7a79bff93b2
Parents: a25cf7b
Author: Yu Li 
Authored: Wed Jul 4 14:21:33 2018 +0800
Committer: Yu Li 
Committed: Thu Jul 5 15:26:56 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |  8 ++-
 .../org/apache/hadoop/hbase/master/HMaster.java |  5 +-
 .../hadoop/hbase/regionserver/wal/FSHLog.java   |  5 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 63 +++
 .../apache/hadoop/hbase/util/TestFSUtils.java   | 64 ++--
 5 files changed, 110 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/50f7894c/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 88a352a..3d1115b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1056,7 +1056,13 @@ public final class HConstants {
* Valid values are: HOT, COLD, WARM, ALL_SSD, ONE_SSD, LAZY_PERSIST
* See 
http://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
   public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
-  public static final String DEFAULT_WAL_STORAGE_POLICY = "HOT";
+  /**
+   * "NONE" is not a valid storage policy and means we defer the policy to 
HDFS. @see
+   * https://issues.apache.org/jira/browse/HBASE-20691";>HBASE-20691
+   */
+  public static final String DEFER_TO_HDFS_STORAGE_POLICY = "NONE";
+  /** By default we defer the WAL storage policy to HDFS */
+  public static final String DEFAULT_WAL_STORAGE_POLICY = 
DEFER_TO_HDFS_STORAGE_POLICY;
 
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =

http://git-wip-us.apache.org/repos/asf/hbase/blob/50f7894c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d228b57..cfa6aa2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1324,8 +1324,9 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
   }
 }
 // Now that it exists, set the log policy
-FSUtils.setStoragePolicy(walFs, conf, walDir, 
HConstants.WAL_STORAGE_POLICY,
-  HConstants.DEFAULT_WAL_STORAGE_POLICY);
+String storagePolicy =
+conf.get(HConstants.WAL_STORAGE_POLICY, 
HConstants.DEFAULT_WAL_STORAGE_POLICY);
+FSUtils.setStoragePolicy(walFs, walDir, storagePolicy);
 
 procedureStore = new WALProcedureStore(conf, walFs, walDir,
 new MasterProcedureEnv.WALStoreLeaseRecovery(this));

http://git-wip-us.apache.org/repos/asf/hbase/blob/50f7894c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 9cef728..141eafb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -469,8 +469,9 @@ public class FSHLog implements WAL {
 }
 // Now that it exists, set the storage policy for the entire directory of 
wal files related to
 // this FSHLog instance
-FSUtils.setStoragePolicy(fs, conf, this.fullPathLogDir, 
HConstants.WAL_STORAGE_POLICY,
-  HConstants.DEFAULT_WAL_STORAGE_POLICY);
+String storagePolicy =
+conf.get(HConstants.WAL_STORAGE_POLICY, 
HConstants.DEFAULT_WAL_STORAGE_POLICY);
+FSUtils.setStoragePolicy(fs, this.fullPathLogDir, storagePolicy);
 this.logFileSuffix = (suffix == null) ? "" : URLEncoder.encode(suffix, 
"UTF8");

hbase git commit: HBASE-20691 Change the default WAL storage policy back to "NONE""

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2.1 60ebdd9fd -> d61bb64e9


HBASE-20691 Change the default WAL storage policy back to "NONE""

This reverts commit 564c193d61cd1f92688a08a3af6d55ce4c4636d8 and added more doc
about why we choose "NONE" as the default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d61bb64e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d61bb64e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d61bb64e

Branch: refs/heads/branch-2.1
Commit: d61bb64e93fc414b131cf2cc5366e18ea80381bc
Parents: 60ebdd9
Author: Yu Li 
Authored: Thu Jun 7 14:12:55 2018 +0800
Committer: Yu Li 
Committed: Wed Jul 4 13:45:54 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |  8 ++-
 .../apache/hadoop/hbase/util/CommonFSUtils.java | 71 +++-
 .../procedure2/store/wal/WALProcedureStore.java |  5 +-
 .../hbase/regionserver/wal/AbstractFSWAL.java   |  5 +-
 .../apache/hadoop/hbase/util/TestFSUtils.java   | 57 +++-
 5 files changed, 105 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d61bb64e/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 5904df6..8e700f9 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1107,7 +1107,13 @@ public final class HConstants {
* Valid values are: HOT, COLD, WARM, ALL_SSD, ONE_SSD, LAZY_PERSIST
* See 
http://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
   public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
-  public static final String DEFAULT_WAL_STORAGE_POLICY = "HOT";
+  /**
+   * "NONE" is not a valid storage policy and means we defer the policy to 
HDFS. @see
+   * https://issues.apache.org/jira/browse/HBASE-20691";>HBASE-20691
+   */
+  public static final String DEFER_TO_HDFS_STORAGE_POLICY = "NONE";
+  /** By default we defer the WAL storage policy to HDFS */
+  public static final String DEFAULT_WAL_STORAGE_POLICY = 
DEFER_TO_HDFS_STORAGE_POLICY;
 
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =

http://git-wip-us.apache.org/repos/asf/hbase/blob/d61bb64e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 5b46de9..76a3601 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -454,36 +454,6 @@ public abstract class CommonFSUtils {
 new Path(namespace)));
   }
 
-  /**
-   * Sets storage policy for given path according to config setting.
-   * If the passed path is a directory, we'll set the storage policy for all 
files
-   * created in the future in said directory. Note that this change in storage
-   * policy takes place at the FileSystem level; it will persist beyond this 
RS's lifecycle.
-   * If we're running on a FileSystem implementation that doesn't support the 
given storage policy
-   * (or storage policies at all), then we'll issue a log message and continue.
-   *
-   * See 
http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
-   *
-   * @param fs We only do anything it implements a setStoragePolicy method
-   * @param conf used to look up storage policy with given key; not modified.
-   * @param path the Path whose storage policy is to be set
-   * @param policyKey Key to use pulling a policy from Configuration:
-   *   e.g. HConstants.WAL_STORAGE_POLICY (hbase.wal.storage.policy).
-   * @param defaultPolicy if the configured policy is equal to this policy 
name, we will skip
-   *   telling the FileSystem to set a storage policy.
-   */
-  public static void setStoragePolicy(final FileSystem fs, final Configuration 
conf,
-  final Path path, final String policyKey, final String defaultPolicy) {
-String storagePolicy = conf.get(policyKey, 
defaultPolicy).toUpperCase(Locale.ROOT);
-if (storagePolicy.equals(defaultPolicy)) {
-  if (LOG.isTraceEnabled()) {
-LOG.trace("default policy of " + defaultPolicy + " requested, exiting 
early.");
-  }
-  return;
-}
-setStoragePolicy(fs, path, storagePolicy);
-  }

hbase git commit: HBASE-20691 Change the default WAL storage policy back to "NONE""

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 341fa5536 -> 4653d4ac6


HBASE-20691 Change the default WAL storage policy back to "NONE""

This reverts commit 564c193d61cd1f92688a08a3af6d55ce4c4636d8 and added more doc
about why we choose "NONE" as the default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4653d4ac
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4653d4ac
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4653d4ac

Branch: refs/heads/branch-2
Commit: 4653d4ac6b5f64eb8d553d4704c47167367c3742
Parents: 341fa55
Author: Yu Li 
Authored: Thu Jun 7 14:12:55 2018 +0800
Committer: Yu Li 
Committed: Wed Jul 4 13:45:16 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |  8 ++-
 .../apache/hadoop/hbase/util/CommonFSUtils.java | 71 +++-
 .../procedure2/store/wal/WALProcedureStore.java |  5 +-
 .../hbase/regionserver/wal/AbstractFSWAL.java   |  5 +-
 .../apache/hadoop/hbase/util/TestFSUtils.java   | 57 +++-
 5 files changed, 105 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4653d4ac/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e2d1bd2..b7d9f4b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1112,7 +1112,13 @@ public final class HConstants {
* Valid values are: HOT, COLD, WARM, ALL_SSD, ONE_SSD, LAZY_PERSIST
* See 
http://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
   public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
-  public static final String DEFAULT_WAL_STORAGE_POLICY = "HOT";
+  /**
+   * "NONE" is not a valid storage policy and means we defer the policy to 
HDFS. @see
+   * https://issues.apache.org/jira/browse/HBASE-20691";>HBASE-20691
+   */
+  public static final String DEFER_TO_HDFS_STORAGE_POLICY = "NONE";
+  /** By default we defer the WAL storage policy to HDFS */
+  public static final String DEFAULT_WAL_STORAGE_POLICY = 
DEFER_TO_HDFS_STORAGE_POLICY;
 
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =

http://git-wip-us.apache.org/repos/asf/hbase/blob/4653d4ac/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 5b46de9..76a3601 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -454,36 +454,6 @@ public abstract class CommonFSUtils {
 new Path(namespace)));
   }
 
-  /**
-   * Sets storage policy for given path according to config setting.
-   * If the passed path is a directory, we'll set the storage policy for all 
files
-   * created in the future in said directory. Note that this change in storage
-   * policy takes place at the FileSystem level; it will persist beyond this 
RS's lifecycle.
-   * If we're running on a FileSystem implementation that doesn't support the 
given storage policy
-   * (or storage policies at all), then we'll issue a log message and continue.
-   *
-   * See 
http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
-   *
-   * @param fs We only do anything it implements a setStoragePolicy method
-   * @param conf used to look up storage policy with given key; not modified.
-   * @param path the Path whose storage policy is to be set
-   * @param policyKey Key to use pulling a policy from Configuration:
-   *   e.g. HConstants.WAL_STORAGE_POLICY (hbase.wal.storage.policy).
-   * @param defaultPolicy if the configured policy is equal to this policy 
name, we will skip
-   *   telling the FileSystem to set a storage policy.
-   */
-  public static void setStoragePolicy(final FileSystem fs, final Configuration 
conf,
-  final Path path, final String policyKey, final String defaultPolicy) {
-String storagePolicy = conf.get(policyKey, 
defaultPolicy).toUpperCase(Locale.ROOT);
-if (storagePolicy.equals(defaultPolicy)) {
-  if (LOG.isTraceEnabled()) {
-LOG.trace("default policy of " + defaultPolicy + " requested, exiting 
early.");
-  }
-  return;
-}
-setStoragePolicy(fs, path, storagePolicy);
-  }
-
 

hbase git commit: HBASE-20691 Change the default WAL storage policy back to "NONE""

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2.0 5b0b20bde -> 4495b10e5


HBASE-20691 Change the default WAL storage policy back to "NONE""

This reverts commit 564c193d61cd1f92688a08a3af6d55ce4c4636d8 and added more doc
about why we choose "NONE" as the default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4495b10e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4495b10e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4495b10e

Branch: refs/heads/branch-2.0
Commit: 4495b10e58cebd0008d620142451f18932bef3aa
Parents: 5b0b20b
Author: Yu Li 
Authored: Thu Jun 7 14:12:55 2018 +0800
Committer: Yu Li 
Committed: Wed Jul 4 13:49:37 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |  8 ++-
 .../apache/hadoop/hbase/util/CommonFSUtils.java | 71 +++-
 .../procedure2/store/wal/WALProcedureStore.java |  5 +-
 .../hbase/regionserver/wal/AbstractFSWAL.java   |  5 +-
 .../apache/hadoop/hbase/util/TestFSUtils.java   | 57 +++-
 5 files changed, 105 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4495b10e/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 7f07450..3937cd5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1080,7 +1080,13 @@ public final class HConstants {
* Valid values are: HOT, COLD, WARM, ALL_SSD, ONE_SSD, LAZY_PERSIST
* See 
http://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
   public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
-  public static final String DEFAULT_WAL_STORAGE_POLICY = "HOT";
+  /**
+   * "NONE" is not a valid storage policy and means we defer the policy to 
HDFS. @see
+   * https://issues.apache.org/jira/browse/HBASE-20691";>HBASE-20691
+   */
+  public static final String DEFER_TO_HDFS_STORAGE_POLICY = "NONE";
+  /** By default we defer the WAL storage policy to HDFS */
+  public static final String DEFAULT_WAL_STORAGE_POLICY = 
DEFER_TO_HDFS_STORAGE_POLICY;
 
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =

http://git-wip-us.apache.org/repos/asf/hbase/blob/4495b10e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 5b46de9..76a3601 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -454,36 +454,6 @@ public abstract class CommonFSUtils {
 new Path(namespace)));
   }
 
-  /**
-   * Sets storage policy for given path according to config setting.
-   * If the passed path is a directory, we'll set the storage policy for all 
files
-   * created in the future in said directory. Note that this change in storage
-   * policy takes place at the FileSystem level; it will persist beyond this 
RS's lifecycle.
-   * If we're running on a FileSystem implementation that doesn't support the 
given storage policy
-   * (or storage policies at all), then we'll issue a log message and continue.
-   *
-   * See 
http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
-   *
-   * @param fs We only do anything it implements a setStoragePolicy method
-   * @param conf used to look up storage policy with given key; not modified.
-   * @param path the Path whose storage policy is to be set
-   * @param policyKey Key to use pulling a policy from Configuration:
-   *   e.g. HConstants.WAL_STORAGE_POLICY (hbase.wal.storage.policy).
-   * @param defaultPolicy if the configured policy is equal to this policy 
name, we will skip
-   *   telling the FileSystem to set a storage policy.
-   */
-  public static void setStoragePolicy(final FileSystem fs, final Configuration 
conf,
-  final Path path, final String policyKey, final String defaultPolicy) {
-String storagePolicy = conf.get(policyKey, 
defaultPolicy).toUpperCase(Locale.ROOT);
-if (storagePolicy.equals(defaultPolicy)) {
-  if (LOG.isTraceEnabled()) {
-LOG.trace("default policy of " + defaultPolicy + " requested, exiting 
early.");
-  }
-  return;
-}
-setStoragePolicy(fs, path, storagePolicy);
-  }

hbase git commit: HBASE-20691 Change the default WAL storage policy back to "NONE""

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master ee3990e42 -> ec8947f22


HBASE-20691 Change the default WAL storage policy back to "NONE""

This reverts commit 564c193d61cd1f92688a08a3af6d55ce4c4636d8 and added more doc
about why we choose "NONE" as the default.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec8947f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec8947f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec8947f2

Branch: refs/heads/master
Commit: ec8947f226d2ad6ab117811bcfa398df012a354d
Parents: ee3990e
Author: Yu Li 
Authored: Thu Jun 7 14:12:55 2018 +0800
Committer: Yu Li 
Committed: Wed Jul 4 13:43:48 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |  8 ++-
 .../apache/hadoop/hbase/util/CommonFSUtils.java | 71 +++-
 .../procedure2/store/wal/WALProcedureStore.java |  5 +-
 .../hbase/regionserver/wal/AbstractFSWAL.java   |  5 +-
 .../apache/hadoop/hbase/util/TestFSUtils.java   | 57 +++-
 5 files changed, 105 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ec8947f2/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index b5a5d62..fa1a772 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1077,7 +1077,13 @@ public final class HConstants {
* Valid values are: HOT, COLD, WARM, ALL_SSD, ONE_SSD, LAZY_PERSIST
* See 
http://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
   public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
-  public static final String DEFAULT_WAL_STORAGE_POLICY = "HOT";
+  /**
+   * "NONE" is not a valid storage policy and means we defer the policy to 
HDFS. @see
+   * https://issues.apache.org/jira/browse/HBASE-20691";>HBASE-20691
+   */
+  public static final String DEFER_TO_HDFS_STORAGE_POLICY = "NONE";
+  /** By default we defer the WAL storage policy to HDFS */
+  public static final String DEFAULT_WAL_STORAGE_POLICY = 
DEFER_TO_HDFS_STORAGE_POLICY;
 
   /** Region in Transition metrics threshold time */
   public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD =

http://git-wip-us.apache.org/repos/asf/hbase/blob/ec8947f2/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
index 5b46de9..76a3601 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java
@@ -454,36 +454,6 @@ public abstract class CommonFSUtils {
 new Path(namespace)));
   }
 
-  /**
-   * Sets storage policy for given path according to config setting.
-   * If the passed path is a directory, we'll set the storage policy for all 
files
-   * created in the future in said directory. Note that this change in storage
-   * policy takes place at the FileSystem level; it will persist beyond this 
RS's lifecycle.
-   * If we're running on a FileSystem implementation that doesn't support the 
given storage policy
-   * (or storage policies at all), then we'll issue a log message and continue.
-   *
-   * See 
http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
-   *
-   * @param fs We only do anything it implements a setStoragePolicy method
-   * @param conf used to look up storage policy with given key; not modified.
-   * @param path the Path whose storage policy is to be set
-   * @param policyKey Key to use pulling a policy from Configuration:
-   *   e.g. HConstants.WAL_STORAGE_POLICY (hbase.wal.storage.policy).
-   * @param defaultPolicy if the configured policy is equal to this policy 
name, we will skip
-   *   telling the FileSystem to set a storage policy.
-   */
-  public static void setStoragePolicy(final FileSystem fs, final Configuration 
conf,
-  final Path path, final String policyKey, final String defaultPolicy) {
-String storagePolicy = conf.get(policyKey, 
defaultPolicy).toUpperCase(Locale.ROOT);
-if (storagePolicy.equals(defaultPolicy)) {
-  if (LOG.isTraceEnabled()) {
-LOG.trace("default policy of " + defaultPolicy + " requested, exiting 
early.");
-  }
-  return;
-}
-setStoragePolicy(fs, path, storagePolicy);
-  }
-
   //

hbase-site git commit: Update link of Chinese website for hbaseconasia2018

[liyu]

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 8952bbd23 -> 8e4ca0283


Update link of Chinese website for hbaseconasia2018


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/8e4ca028
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/8e4ca028
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/8e4ca028

Branch: refs/heads/asf-site
Commit: 8e4ca028390b8c0ce775d031ec4c5904b0dc24d7
Parents: 8952bbd
Author: Yu Li 
Authored: Thu Jun 7 15:21:37 2018 +0800
Committer: 绝顶 
Committed: Thu Jun 7 15:23:27 2018 +0800

--
 hbaseconasia-2018/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8e4ca028/hbaseconasia-2018/index.html
--
diff --git a/hbaseconasia-2018/index.html b/hbaseconasia-2018/index.html
index a6c2158..55f4fc3 100644
--- a/hbaseconasia-2018/index.html
+++ b/hbaseconasia-2018/index.html
@@ -73,7 +73,7 @@
 
 Contact
 
-https://www.aliyun.com/hbaseconasia2018";>中文网站
+https://m.aliyun.com/markets/aliyun/hbaseconasia2018";>中文网站
 
 
 

hbase-site git commit: Update English-version registration link for hbaseconasia

[liyu]

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site b5d188bf3 -> 8a9782b65


Update English-version registration link for hbaseconasia


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/8a9782b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/8a9782b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/8a9782b6

Branch: refs/heads/asf-site
Commit: 8a9782b65457eb9d73fa09dbaf4c9f5cb8731833
Parents: b5d188b
Author: Yu Li 
Authored: Tue May 22 21:41:05 2018 +0800
Committer: 绝顶 
Committed: Tue May 22 21:41:53 2018 +0800

--
 hbaseconasia-2018/index.html | 15 ++-
 1 file changed, 2 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8a9782b6/hbaseconasia-2018/index.html
--
diff --git a/hbaseconasia-2018/index.html b/hbaseconasia-2018/index.html
index b81e10d..a6c2158 100644
--- a/hbaseconasia-2018/index.html
+++ b/hbaseconasia-2018/index.html
@@ -161,26 +161,15 @@



-
- Registration for HBaseConAsia 2018 will be open soon, please 
watch and wait for the registration link.


-

 
 

hbase-site git commit: Update logo and add Chinese website link (unavailable yet) to hbaseconasia

[liyu]

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 96e2b2956 -> b5d188bf3


Update logo and add Chinese website link (unavailable yet) to hbaseconasia


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/b5d188bf
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/b5d188bf
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/b5d188bf

Branch: refs/heads/asf-site
Commit: b5d188bf3ef4f048ed8411c150487f473c0f1b21
Parents: 96e2b29
Author: Yu Li 
Authored: Mon May 21 17:03:53 2018 +0800
Committer: 绝顶 
Committed: Mon May 21 17:06:57 2018 +0800

--
 hbaseconasia-2018/img/hbca.png | Bin 0 -> 12509 bytes
 hbaseconasia-2018/index.html   |   3 ++-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/b5d188bf/hbaseconasia-2018/img/hbca.png
--
diff --git a/hbaseconasia-2018/img/hbca.png b/hbaseconasia-2018/img/hbca.png
new file mode 100644
index 000..bdf6525
Binary files /dev/null and b/hbaseconasia-2018/img/hbca.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/b5d188bf/hbaseconasia-2018/index.html
--
diff --git a/hbaseconasia-2018/index.html b/hbaseconasia-2018/index.html
index 3760581..b81e10d 100644
--- a/hbaseconasia-2018/index.html
+++ b/hbaseconasia-2018/index.html
@@ -73,6 +73,7 @@
 
 Contact
 
+https://www.aliyun.com/hbaseconasia2018";>中文网站
 
 
 
@@ -85,7 +86,7 @@
 
 
 
-
+
 
 
 

[7/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.svg
--
diff --git a/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.svg 
b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.svg
new file mode 100755
index 000..a9f8469
--- /dev/null
+++ b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.svg
@@ -0,0 +1,504 @@
+
+http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"; >
+http://www.w3.org/2000/svg";>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.ttf
--
diff --git a/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.ttf 
b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.ttf
new file mode 100755
index 000..5cd6cff
Binary files /dev/null and 
b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.ttf differ

[3/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/js/jquery-1.11.0.js
--
diff --git a/hbaseconasia-2018/js/jquery-1.11.0.js 
b/hbaseconasia-2018/js/jquery-1.11.0.js
new file mode 100755
index 000..046e93a
--- /dev/null
+++ b/hbaseconasia-2018/js/jquery-1.11.0.js
@@ -0,0 +1,4 @@
+/*! jQuery v1.11.0 | (c) 2005, 2014 jQuery Foundation, Inc. | 
jquery.org/license */
+!function(a,b){"object"==typeof module&&"object"==typeof 
module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw
 new Error("jQuery requires a window with a document");return 
b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var 
c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k="".trim,l={},m="1.11.0",n=function(a,b){return
 new 
n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return
 
b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return
 d.call(this)},get:function(a){return 
null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var 
b=n.merge(this.constructor(),a);return 
b.prevObject=this,b.context=this.context,b},each:function(a,b){return 
n.each(this,a,b)},map:function(a){return 
this.pushStack(n.map(this,function(b,c){return 
a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(th
 is,arguments))},first:function(){return this.eq(0)},last:function(){return 
this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return 
this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return 
this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var
 
a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof
 g&&(j=g,g=arguments[h]||{},h++),"object"==typeof 
g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d
 in 
e)a=g[d],c=e[d],g!==c&&(j&&c&&(n.isPlainObject(c)||(b=n.isArray(c)))?(b?(b=!1,f=a&&n.isArray(a)?a:[]):f=a&&n.isPlainObject(a)?a:{},g[d]=n.extend(j,f,c)):void
 0!==c&&(g[d]=c));return 
g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw
 new 
Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray||function(a){return"array"===n.type(a)},isWindow:function(a){return
 n
 ull!=a&&a==a.window},isNumeric:function(a){return 
a-parseFloat(a)>=0},isEmptyObject:function(a){var b;for(b in 
a)return!1;return!0},isPlainObject:function(a){var 
b;if(!a||"object"!==n.type(a)||a.nodeType||n.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(l.ownLast)for(b
 in a)return j.call(a,b);for(b in a);return void 
0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof 
a||"function"==typeof a?h[i.call(a)]||"object":typeof 
a},globalEval:function(b){b&&n.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return
 a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return 
a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var
 
d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else
 for(e in a)if(d=b.apply(a[e],c),d===!1)break}else 
if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!
 1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return 
a},trim:k&&!k.call("\ufeff\xa0")?function(a){return 
null==a?"":k.call(a)}:function(a){return 
null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return 
null!=a&&(s(Object(a))?n.merge(c,"string"==typeof 
a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return 
g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in 
b&&b[c]===a)return c}return-1},merge:function(a,b){var 
c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 
0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var 
d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return 
e},map:function(a,b,c){var 
d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else
 for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return 
e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof 
b&&(f=a[b],b=a,a=f),n.isFunction(a)?(c=d.call(argum
 ents,2),e=function(){return 
a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||n.guid++,e):void
 0},now:function(){return+new Date},support:l}),n.each("Boolean Number String 
Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object 
"+b+"]"]=b.toLowerCase()});function s(a){var 
b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof
 b&&b>0&&b-1 in a}var t=function(a){var

[6/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.woff
--
diff --git a/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.woff 
b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.woff
new file mode 100755
index 000..9eaecb3
Binary files /dev/null and 
b/hbaseconasia-2018/css/font-awesome/fonts/fontawesome-webfont.woff differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/css/hbasecon.css
--
diff --git a/hbaseconasia-2018/css/hbasecon.css 
b/hbaseconasia-2018/css/hbasecon.css
new file mode 100644
index 000..1cfff6f
--- /dev/null
+++ b/hbaseconasia-2018/css/hbasecon.css
@@ -0,0 +1,3 @@
+.navbar-default {
+  background-color: #ba160c;
+}

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/feed.xml
--
diff --git a/hbaseconasia-2018/feed.xml b/hbaseconasia-2018/feed.xml
new file mode 100644
index 000..cb8b56c
--- /dev/null
+++ b/hbaseconasia-2018/feed.xml
@@ -0,0 +1,22 @@
+
+
+http://www.w3.org/2005/Atom"; >
+  https://jekyllrb.com/"; version="3.8.1">Jekyll
+  https://hbase.apache.org/hbaseconasia-2018//hbaseconasia-2018/feed.xml"; 
rel="self" type="application/atom+xml" />
+  https://hbase.apache.org/hbaseconasia-2018//hbaseconasia-2018/"; 
rel="alternate" type="text/html" />
+  2018-05-16T19:40:03+08:00
+  https://hbase.apache.org/hbaseconasia-2018//hbaseconasia-2018/
+
+  
+HBaseConAsia
+  
+
+  
+The Community Event for Apache HBase™
+  
+
+  
+
+  
+  
+

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/hbase.png
--
diff --git a/hbaseconasia-2018/img/hbase.png b/hbaseconasia-2018/img/hbase.png
new file mode 100755
index 000..e91eb8d
Binary files /dev/null and b/hbaseconasia-2018/img/hbase.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/hbasecon2018.png
--
diff --git a/hbaseconasia-2018/img/hbasecon2018.png 
b/hbaseconasia-2018/img/hbasecon2018.png
new file mode 100644
index 000..c41fa8f
Binary files /dev/null and b/hbaseconasia-2018/img/hbasecon2018.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/hbaseconasia2018.png
--
diff --git a/hbaseconasia-2018/img/hbaseconasia2018.png 
b/hbaseconasia-2018/img/hbaseconasia2018.png
new file mode 100644
index 000..2b27e3c
Binary files /dev/null and b/hbaseconasia-2018/img/hbaseconasia2018.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/jumping-orca_rotated_12percent.png
--
diff --git a/hbaseconasia-2018/img/jumping-orca_rotated_12percent.png 
b/hbaseconasia-2018/img/jumping-orca_rotated_12percent.png
new file mode 100644
index 000..1942f9a
Binary files /dev/null and 
b/hbaseconasia-2018/img/jumping-orca_rotated_12percent.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/phoenix.png
--
diff --git a/hbaseconasia-2018/img/phoenix.png 
b/hbaseconasia-2018/img/phoenix.png
new file mode 100644
index 000..d5f32ec
Binary files /dev/null and b/hbaseconasia-2018/img/phoenix.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/sponsors/alibaba.png
--
diff --git a/hbaseconasia-2018/img/sponsors/alibaba.png 
b/hbaseconasia-2018/img/sponsors/alibaba.png
new file mode 100644
index 000..60129fd
Binary files /dev/null and b/hbaseconasia-2018/img/sponsors/alibaba.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/sponsors/asf.png
--
diff --git a/hbaseconasia-2018/img/sponsors/asf.png 
b/hbaseconasia-2018/img/sponsors/asf.png
new file mode 100644
index 000..b20bb7f
Binary files /dev/null and b/hbaseconasia-2018/img/sponsors/asf.png differ

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/img/sponsors/hortonworks.svg
--
diff --git a/hbaseconasia-2018/img/sponsors/hortonworks.svg 
b/hbaseconasia-2018/img/sponsors/hortonworks.svg
new file mode 100644
index 000..c5888b0
--- /dev/null
+++ b/hbaseconasia-2018/img/sponsors/hortonworks.svg
@@ -0,0 +1 @@
+http://www.w3.org/2000/svg"; 
xmlns:x

[5/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/js/bootstrap.js
--
diff --git a/hbaseconasia-2018/js/bootstrap.js 
b/hbaseconasia-2018/js/bootstrap.js
new file mode 100755
index 000..53da1c7
--- /dev/null
+++ b/hbaseconasia-2018/js/bootstrap.js
@@ -0,0 +1,2114 @@
+/*!
+ * Bootstrap v3.2.0 (http://getbootstrap.com)
+ * Copyright 2011-2014 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */
+
+if (typeof jQuery === 'undefined') { throw new Error('Bootstrap\'s JavaScript 
requires jQuery') }
+
+/* 
+ * Bootstrap: transition.js v3.2.0
+ * http://getbootstrap.com/javascript/#transitions
+ * 
+ * Copyright 2011-2014 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ *  */
+
+
++function ($) {
+  'use strict';
+
+  // CSS TRANSITION SUPPORT (Shoutout: http://www.modernizr.com/)
+  // 
+
+  function transitionEnd() {
+var el = document.createElement('bootstrap')
+
+var transEndEventNames = {
+  WebkitTransition : 'webkitTransitionEnd',
+  MozTransition: 'transitionend',
+  OTransition  : 'oTransitionEnd otransitionend',
+  transition   : 'transitionend'
+}
+
+for (var name in transEndEventNames) {
+  if (el.style[name] !== undefined) {
+return { end: transEndEventNames[name] }
+  }
+}
+
+return false // explicit for ie8 (  ._.)
+  }
+
+  // http://blog.alexmaccaw.com/css-transitions
+  $.fn.emulateTransitionEnd = function (duration) {
+var called = false
+var $el = this
+$(this).one('bsTransitionEnd', function () { called = true })
+var callback = function () { if (!called) 
$($el).trigger($.support.transition.end) }
+setTimeout(callback, duration)
+return this
+  }
+
+  $(function () {
+$.support.transition = transitionEnd()
+
+if (!$.support.transition) return
+
+$.event.special.bsTransitionEnd = {
+  bindType: $.support.transition.end,
+  delegateType: $.support.transition.end,
+  handle: function (e) {
+if ($(e.target).is(this)) return e.handleObj.handler.apply(this, 
arguments)
+  }
+}
+  })
+
+}(jQuery);
+
+/* 
+ * Bootstrap: alert.js v3.2.0
+ * http://getbootstrap.com/javascript/#alerts
+ * 
+ * Copyright 2011-2014 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ *  */
+
+
++function ($) {
+  'use strict';
+
+  // ALERT CLASS DEFINITION
+  // ==
+
+  var dismiss = '[data-dismiss="alert"]'
+  var Alert   = function (el) {
+$(el).on('click', dismiss, this.close)
+  }
+
+  Alert.VERSION = '3.2.0'
+
+  Alert.prototype.close = function (e) {
+var $this= $(this)
+var selector = $this.attr('data-target')
+
+if (!selector) {
+  selector = $this.attr('href')
+  selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip 
for ie7
+}
+
+var $parent = $(selector)
+
+if (e) e.preventDefault()
+
+if (!$parent.length) {
+  $parent = $this.hasClass('alert') ? $this : $this.parent()
+}
+
+$parent.trigger(e = $.Event('close.bs.alert'))
+
+if (e.isDefaultPrevented()) return
+
+$parent.removeClass('in')
+
+function removeElement() {
+  // detach from parent, fire event then clean up data
+  $parent.detach().trigger('closed.bs.alert').remove()
+}
+
+$.support.transition && $parent.hasClass('fade') ?
+  $parent
+.one('bsTransitionEnd', removeElement)
+.emulateTransitionEnd(150) :
+  removeElement()
+  }
+
+
+  // ALERT PLUGIN DEFINITION
+  // ===
+
+  function Plugin(option) {
+return this.each(function () {
+  var $this = $(this)
+  var data  = $this.data('bs.alert')
+
+  if (!data) $this.data('bs.alert', (data = new Alert(this)))
+  if (typeof option == 'string') data[option].call($this)
+})
+  }
+
+  var old = $.fn.alert
+
+  $.fn.alert = Plugin
+  $.fn.alert.Constructor = Alert
+
+
+  // ALERT NO CONFLICT
+  // =
+
+  $.fn.alert.noConflict = function () {
+$.fn.alert = old
+return this
+  }
+
+
+  // ALERT DATA-API
+  // ==
+
+  $(document).on('click.bs.alert.data-api', dismiss, Alert.prototype.close)
+
+}(jQuery);
+
+/* 
+ * Bootstrap: button.js v3.2.0
+ * http://getbootstrap.c

[8/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

Add in the custom hbasecon asia 2018 website


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/c8db7dc9
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/c8db7dc9
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/c8db7dc9

Branch: refs/heads/asf-site
Commit: c8db7dc9ce79f13e9499130e3b1e65b4c2317344
Parents: 662611a
Author: Yu Li 
Authored: Wed May 16 19:47:04 2018 +0800
Committer: 绝顶 
Committed: Wed May 16 19:48:08 2018 +0800

--
 hbaseconasia-2018/LICENCE   |  201 ++
 hbaseconasia-2018/README.md |   13 +
 .../css/font-awesome/css/font-awesome.css   | 1566 +
 .../css/font-awesome/css/font-awesome.min.css   |4 +
 .../css/font-awesome/fonts/FontAwesome.otf  |  Bin 0 -> 75188 bytes
 .../font-awesome/fonts/fontawesome-webfont.eot  |  Bin 0 -> 72449 bytes
 .../font-awesome/fonts/fontawesome-webfont.svg  |  504 +
 .../font-awesome/fonts/fontawesome-webfont.ttf  |  Bin 0 -> 141564 bytes
 .../font-awesome/fonts/fontawesome-webfont.woff |  Bin 0 -> 83760 bytes
 hbaseconasia-2018/css/hbasecon.css  |3 +
 hbaseconasia-2018/feed.xml  |   22 +
 hbaseconasia-2018/img/hbase.png |  Bin 0 -> 21196 bytes
 hbaseconasia-2018/img/hbasecon2018.png  |  Bin 0 -> 12145 bytes
 hbaseconasia-2018/img/hbaseconasia2018.png  |  Bin 0 -> 10907 bytes
 .../img/jumping-orca_rotated_12percent.png  |  Bin 0 -> 2401 bytes
 hbaseconasia-2018/img/phoenix.png   |  Bin 0 -> 9552 bytes
 hbaseconasia-2018/img/sponsors/alibaba.png  |  Bin 0 -> 54770 bytes
 hbaseconasia-2018/img/sponsors/asf.png  |  Bin 0 -> 21243 bytes
 hbaseconasia-2018/img/sponsors/hortonworks.svg  |1 +
 hbaseconasia-2018/img/sponsors/siftscience.png  |  Bin 0 -> 53828 bytes
 hbaseconasia-2018/index.html|  345 +++
 hbaseconasia-2018/js/bootstrap.js   | 2114 ++
 hbaseconasia-2018/js/bootstrap.min.js   |6 +
 hbaseconasia-2018/js/cbpAnimatedHeader.js   |   44 +
 hbaseconasia-2018/js/cbpAnimatedHeader.min.js   |   11 +
 hbaseconasia-2018/js/classie.js |   80 +
 hbaseconasia-2018/js/freelancer.js  |   37 +
 hbaseconasia-2018/js/jqBootstrapValidation.js   |  912 
 hbaseconasia-2018/js/jquery-1.11.0.js   |4 +
 hbaseconasia-2018/js/jquery.easing.min.js   |   44 +
 hbaseconasia-2018/style.css |  484 
 31 files changed, 6395 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/LICENCE
--
diff --git a/hbaseconasia-2018/LICENCE b/hbaseconasia-2018/LICENCE
new file mode 100644
index 000..8dada3e
--- /dev/null
+++ b/hbaseconasia-2018/LICENCE
@@ -0,0 +1,201 @@
+ Apache License
+   Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any w

[4/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/js/bootstrap.min.js
--
diff --git a/hbaseconasia-2018/js/bootstrap.min.js 
b/hbaseconasia-2018/js/bootstrap.min.js
new file mode 100755
index 000..7c1561a
--- /dev/null
+++ b/hbaseconasia-2018/js/bootstrap.min.js
@@ -0,0 +1,6 @@
+/*!
+ * Bootstrap v3.2.0 (http://getbootstrap.com)
+ * Copyright 2011-2014 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */
+if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires 
jQuery");+function(a){"use strict";function b(){var 
a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd
 otransitionend",transition:"transitionend"};for(var c in b)if(void 
0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var
 c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var 
e=function(){c||a(d).trigger(a.support.transition.end)};return 
setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return
 a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 
0}})})}(jQuery),+function(a){"use strict";function b(b){return 
this.each(function(){var 
c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new 
d(this)),"string"==type
 of b&&e[b].call(c)})}var 
c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.2.0",d.prototype.close=function(b){function
 c(){f.detach().trigger("closed.bs.alert").remove()}var 
d=a(this),e=d.attr("data-target");e||(e=d.attr("href"),e=e&&e.replace(/.*(?=#[^\s]*$)/,""));var
 
f=a(e);b&&b.preventDefault(),f.length||(f=d.hasClass("alert")?d:d.parent()),f.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",c).emulateTransitionEnd(150):c())};var
 
e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return
 
a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use
 strict";function b(b){return this.each(function(){var 
d=a(this),e=d.data("bs.button"),f="object"==typeof 
b&&b;e||d.data("bs.button",e=new 
c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var 
c=function(b,d){this.$element=a(b)
 
,this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.2.0",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var
 
c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),d[e](null==f[b]?this.options[b]:f[b]),setTimeout(a.proxy(function(){"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var
 a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var 
c=this.$element.find("input");"radio"==c.prop("type")&&(c.prop("checked")&&this.$element.hasClass("active")?a=!1:b.find(".active").removeClass("active")),a&&c.prop("checked",!this.$element.hasClass("active")).trigger("change")}a&&this.$element.toggleClass("active")};var
 
d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return
 a.fn.button=d,this},a(document).on("clic
 k.bs.button.data-api",'[data-toggle^="button"]',function(c){var 
d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),c.preventDefault()})}(jQuery),+function(a){"use
 strict";function b(b){return this.each(function(){var 
d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof
 b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new 
c(this,f)),"number"==typeof 
b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var 
c=function(b,c){this.$element=a(b).on("keydown.bs.carousel",a.proxy(this.keydown,this)),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=this.sliding=this.interval=this.$active=this.$items=null,"hover"==this.options.pause&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.2.0",c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0},c.prototype.keydown=function(a){switch(a.which){case
 37:this.prev();brea
 k;case 
39:this.next();break;default:return}a.preventDefault()},c.prototype.cycle=function(b){return
 
b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return
 
this.$items=a.parent().children(".item"),this.$items.index(a||this.$acti

[1/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 662611ad4 -> c8db7dc9c


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/style.css
--
diff --git a/hbaseconasia-2018/style.css b/hbaseconasia-2018/style.css
new file mode 100644
index 000..e0514ef
--- /dev/null
+++ b/hbaseconasia-2018/style.css
@@ -0,0 +1,484 @@
+/*!
+ * Bootswatch v3.2.0
+ * Homepage: http://bootswatch.com
+ * Copyright 2012-2014 Thomas Park
+ * Licensed under MIT
+ * Based on Bootstrap
+*//*! normalize.css v3.0.1 | MIT License | git.io/normalize 
*/html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px
 
dotted}b,strong{font-weight:bold}dfn{font-style:italic}h1{font-size:2em;margin:0.67em
 
0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em
 
40px}hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,
 monospace;font-size:1em}button,input,optgroup,select,textarea{colo
 
r:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html
 
input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer}button[disabled],html
 
input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0}input[type="number"]::-webkit-inner-spin-button,input[type="number"]::-webkit-outer-spin-button{height:auto}input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px
 solid #c0c0c0;margin:0 2px;padding:0.35em 0.625em 
0.75em}legend{border:0;padding:0}textarea{overflow:auto}optgroup{font-weight:bold}table{border-collapse:collapse;border-spacing:0}td,th{pa
 dding:0}@media print{*{text-shadow:none !important;color:#000 
!important;background:transparent !important;box-shadow:none 
!important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" 
attr(href) ")"}abbr[title]:after{content:" (" attr(title) 
")"}a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px
 solid 
#999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100%
 
!important}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}select{background:#fff
 !important}.navbar{display:none}.table td,.table th{background-color:#fff 
!important}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000 
!important}.label{border:1px solid #000}.table{border-collapse:collapse 
!important}.table-bordered th,.table-bordered td{border:1px solid #ddd 
!important}}@font-face{font-family:'Glyphicons 
Halflings';src:url('../fonts/glyphicons-halflings-regular.eot');src:url('../fonts/glyphicons-halflings-regular.eot
 ?#iefix') 
format('embedded-opentype'),url('../fonts/glyphicons-halflings-regular.woff') 
format('woff'),url('../fonts/glyphicons-halflings-regular.ttf') 
format('truetype'),url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular')
 
format('svg')}.glyphicon{position:relative;top:1px;display:inline-block;font-family:'Glyphicons
 
Halflings';font-style:normal;font-weight:normal;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.glyphicon-asterisk:before{content:"\2a"}.glyphicon-plus:before{content:"\2b"}.glyphicon-euro:before{content:"\20ac"}.glyphicon-minus:before{content:"\2212"}.glyphicon-cloud:before{content:"\2601"}.glyphicon-envelope:before{content:"\2709"}.glyphicon-pencil:before{content:"\270f"}.glyphicon-glass:before{content:"\e001"}.glyphicon-music:before{content:"\e002"}.glyphicon-search:before{content:"\e003"}.glyphicon-heart:before{content:"\e005"}.glyphicon-star:before{content:"\e006"}.glyphicon-star-empty:before{content:"\e0
 
07"}.glyphicon-user:before{content:"\e008"}.glyphicon-film:before{content:"\e009"}.glyphicon-th-large:before{content:"\e010"}.glyphicon-th:before{content:"\e011"}.glyphicon-th-list:before{content:"\e012"}.glyphicon-ok:before{content:"\e013"}.glyphicon-remove:before{content:"\e014"}.glyphicon-zoom-in:before{content:"\e015"}.glyphicon-zoom-out:before{content:"\e016"}.glyphicon-off:before

[2/8] hbase-site git commit: Add in the custom hbasecon asia 2018 website

[liyu]

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c8db7dc9/hbaseconasia-2018/js/jquery.easing.min.js
--
diff --git a/hbaseconasia-2018/js/jquery.easing.min.js 
b/hbaseconasia-2018/js/jquery.easing.min.js
new file mode 100644
index 000..4a54fcd
--- /dev/null
+++ b/hbaseconasia-2018/js/jquery.easing.min.js
@@ -0,0 +1,44 @@
+/*
+ * jQuery Easing v1.3 - http://gsgd.co.uk/sandbox/jquery/easing/
+ *
+ * Uses the built in easing capabilities added In jQuery 1.1
+ * to offer multiple easing options
+ *
+ * TERMS OF USE - EASING EQUATIONS
+ *
+ * Open source under the BSD License.
+ *
+ * Copyright © 2001 Robert Penner
+ * All rights reserved.
+ *
+ * TERMS OF USE - jQuery Easing
+ *
+ * Open source under the BSD License.
+ *
+ * Copyright © 2008 George McGinley Smith
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without 
modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this 
list of
+ * conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, 
this list
+ * of conditions and the following disclaimer in the documentation and/or 
other materials
+ * provided with the distribution.
+ *
+ * Neither the name of the author nor the names of contributors may be used to 
endorse
+ * or promote products derived from this software without specific prior 
written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 
EVENT SHALL THE
+ *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
INCIDENTAL, SPECIAL,
+ *  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
PROCUREMENT OF SUBSTITUTE
+ *  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 
TORT (INCLUDING
+ *  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+*/
+jQuery.easing.jswing=jQuery.easing.swing;jQuery.extend(jQuery.easing,{def:"easeOutQuad",swing:function(e,f,a,h,g){return
 
jQuery.easing[jQuery.easing.def](e,f,a,h,g)},easeInQuad:function(e,f,a,h,g){return
 h*(f/=g)*f+a},easeOutQuad:function(e,f,a,h,g){return 
-h*(f/=g)*(f-2)+a},easeInOutQuad:function(e,f,a,h,g){if((f/=g/2)<1){return 
h/2*f*f+a}return -h/2*((--f)*(f-2)-1)+a},easeInCubic:function(e,f,a,h,g){return 
h*(f/=g)*f*f+a},easeOutCubic:function(e,f,a,h,g){return 
h*((f=f/g-1)*f*f+1)+a},easeInOutCubic:function(e,f,a,h,g){if((f/=g/2)<1){return 
h/2*f*f*f+a}return h/2*((f-=2)*f*f+2)+a},easeInQuart:function(e,f,a,h,g){return 
h*(f/=g)*f*f*f+a},easeOutQuart:function(e,f,a,h,g){return 
-h*((f=f/g-1)*f*f*f-1)+a},easeInOutQuart:function(e,f,a,h,g){if((f/=g/2)<1){return
 h/2*f*f*f*f+a}return 
-h/2*((f-=2)*f*f*f-2)+a},easeInQuint:function(e,f,a,h,g){return 
h*(f/=g)*f*f*f*f+a},easeOutQuint:function(e,f,a,h,g){return 
h*((f=f/g-1)*f*f*f*f+1)+a},easeInOutQuint:function(e,f,a,h,g){if((f/=g/2)<1){return
  h/2*f*f*f*f*f+a}return 
h/2*((f-=2)*f*f*f*f+2)+a},easeInSine:function(e,f,a,h,g){return 
-h*Math.cos(f/g*(Math.PI/2))+h+a},easeOutSine:function(e,f,a,h,g){return 
h*Math.sin(f/g*(Math.PI/2))+a},easeInOutSine:function(e,f,a,h,g){return 
-h/2*(Math.cos(Math.PI*f/g)-1)+a},easeInExpo:function(e,f,a,h,g){return(f==0)?a:h*Math.pow(2,10*(f/g-1))+a},easeOutExpo:function(e,f,a,h,g){return(f==g)?a+h:h*(-Math.pow(2,-10*f/g)+1)+a},easeInOutExpo:function(e,f,a,h,g){if(f==0){return
 a}if(f==g){return a+h}if((f/=g/2)<1){return h/2*Math.pow(2,10*(f-1))+a}return 
h/2*(-Math.pow(2,-10*--f)+2)+a},easeInCirc:function(e,f,a,h,g){return 
-h*(Math.sqrt(1-(f/=g)*f)-1)+a},easeOutCirc:function(e,f,a,h,g){return 
h*Math.sqrt(1-(f=f/g-1)*f)+a},easeInOutCirc:function(e,f,a,h,g){if((f/=g/2)<1){return
 -h/2*(Math.sqrt(1-f*f)-1)+a}return 
h/2*(Math.sqrt(1-(f-=2)*f)+1)+a},easeInElastic:function(f,h,e,l,k){var 
i=1.70158;var j=0;var g=l;if(h==0){return e}if((h/=k)==1){return 
e+l}if(!j){j=k*0.3}if(g

hbase git commit: HBASE-20419 Fix potential NPE in ZKUtil#listChildrenAndWatchForNewChildren callers

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 4fea86296 -> 1cb05a18b


HBASE-20419 Fix potential NPE in ZKUtil#listChildrenAndWatchForNewChildren 
callers

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1cb05a18
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1cb05a18
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1cb05a18

Branch: refs/heads/branch-2
Commit: 1cb05a18bc1564585ce9c03680b56349672cd709
Parents: 4fea862
Author: lujie 
Authored: Mon Apr 16 16:01:49 2018 +0800
Committer: Yu Li 
Committed: Mon Apr 16 16:07:59 2018 +0800

--
 .../apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java  | 6 +-
 .../apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java | 8 ++--
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1cb05a18/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
--
diff --git 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 9294255..d6618bf 100644
--- 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -353,7 +353,11 @@ final class RSGroupInfoManagerImpl implements 
RSGroupInfoManager {
 //Overwrite any info stored by table, this takes precedence
 try {
   if(ZKUtil.checkExists(watcher, groupBasePath) != -1) {
-for(String znode: ZKUtil.listChildrenAndWatchForNewChildren(watcher, 
groupBasePath)) {
+List children = 
ZKUtil.listChildrenAndWatchForNewChildren(watcher, groupBasePath);
+if (children == null) {
+  return RSGroupInfoList;
+}
+for(String znode: children) {
   byte[] data = ZKUtil.getData(watcher, 
ZNodePaths.joinZNode(groupBasePath, znode));
   if(data.length > 0) {
 ProtobufUtil.expectPBMagicPrefix(data);

http://git-wip-us.apache.org/repos/asf/hbase/blob/1cb05a18/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index f29d133..57d929d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -135,8 +135,12 @@ public class ZKProcedureMemberRpcs implements 
ProcedureMemberRpcs {
 LOG.debug("Checking for aborted procedures on node: '" + 
zkController.getAbortZnode() + "'");
 try {
   // this is the list of the currently aborted procedues
-  for (String node : 
ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
-zkController.getAbortZnode())) {
+  List children = 
ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+   zkController.getAbortZnode());
+  if (children == null || children.isEmpty()) {
+return;
+  }
+  for (String node : children) {
 String abortNode = ZNodePaths.joinZNode(zkController.getAbortZnode(), 
node);
 abort(abortNode);
   }

hbase git commit: HBASE-20419 Fix potential NPE in ZKUtil#listChildrenAndWatchForNewChildren callers

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 125767b44 -> 1339ff966


HBASE-20419 Fix potential NPE in ZKUtil#listChildrenAndWatchForNewChildren 
callers

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1339ff96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1339ff96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1339ff96

Branch: refs/heads/master
Commit: 1339ff9666f449d8850236802c008a27307b7c61
Parents: 125767b
Author: lujie 
Authored: Mon Apr 16 16:01:49 2018 +0800
Committer: Yu Li 
Committed: Mon Apr 16 16:06:57 2018 +0800

--
 .../apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java  | 6 +-
 .../apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java | 8 ++--
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1339ff96/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
--
diff --git 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index a9f8284..659343c 100644
--- 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -354,7 +354,11 @@ final class RSGroupInfoManagerImpl implements 
RSGroupInfoManager {
 //Overwrite any info stored by table, this takes precedence
 try {
   if(ZKUtil.checkExists(watcher, groupBasePath) != -1) {
-for(String znode: ZKUtil.listChildrenAndWatchForNewChildren(watcher, 
groupBasePath)) {
+List children = 
ZKUtil.listChildrenAndWatchForNewChildren(watcher, groupBasePath);
+if (children == null) {
+  return RSGroupInfoList;
+}
+for(String znode: children) {
   byte[] data = ZKUtil.getData(watcher, 
ZNodePaths.joinZNode(groupBasePath, znode));
   if(data.length > 0) {
 ProtobufUtil.expectPBMagicPrefix(data);

http://git-wip-us.apache.org/repos/asf/hbase/blob/1339ff96/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index f29d133..57d929d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -135,8 +135,12 @@ public class ZKProcedureMemberRpcs implements 
ProcedureMemberRpcs {
 LOG.debug("Checking for aborted procedures on node: '" + 
zkController.getAbortZnode() + "'");
 try {
   // this is the list of the currently aborted procedues
-  for (String node : 
ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
-zkController.getAbortZnode())) {
+  List children = 
ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
+   zkController.getAbortZnode());
+  if (children == null || children.isEmpty()) {
+return;
+  }
+  for (String node : children) {
 String abortNode = ZNodePaths.joinZNode(zkController.getAbortZnode(), 
node);
 abort(abortNode);
   }

hbase git commit: HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in CleanerChore) to branch-1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.4 3c98b3d63 -> 24c296e5c


HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in 
CleanerChore) to branch-1

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/24c296e5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/24c296e5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/24c296e5

Branch: refs/heads/branch-1.4
Commit: 24c296e5c644be931e3053664009f31c858305be
Parents: 3c98b3d
Author: Reid Chan 
Authored: Wed Apr 11 14:16:08 2018 +0800
Committer: Yu Li 
Committed: Wed Apr 11 19:02:31 2018 +0800

--
 .../org/apache/hadoop/hbase/master/HMaster.java |   4 +
 .../hbase/master/cleaner/CleanerChore.java  | 401 ++-
 .../hadoop/hbase/master/cleaner/LogCleaner.java | 181 +
 .../TestZooKeeperTableArchiveClient.java|   3 +
 .../hbase/master/cleaner/TestCleanerChore.java  |  99 -
 .../hbase/master/cleaner/TestHFileCleaner.java  |   1 +
 .../master/cleaner/TestHFileLinkCleaner.java|   1 +
 .../hbase/master/cleaner/TestLogsCleaner.java   |  57 +++
 8 files changed, 647 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/24c296e5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0f2b8e6..80bda8d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
 import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
+import org.apache.hadoop.hbase.master.cleaner.CleanerChore;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore;
@@ -892,6 +893,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
(System.currentTimeMillis() - masterActiveTime) / 1000.0f));
 this.masterFinishedInitializationTime = System.currentTimeMillis();
 configurationManager.registerObserver(this.balancer);
+configurationManager.registerObserver(this.logCleaner);
 
 // Set master as 'initialized'.
 setInitialized(true);
@@ -1214,6 +1216,8 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
startProcedureExecutor();
 
+// Initial cleaner chore
+CleanerChore.initChorePool(conf);
// Start log cleaner thread
int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 
1000);
this.logCleaner =

http://git-wip-us.apache.org/repos/asf/hbase/blob/24c296e5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 743d9a4..50d8da3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.hbase.master.cleaner;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinTask;
+import java.util.concurrent.RecursiveTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -28,12 +32,16 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FileStatusFilter;
+import org.apache.hadoop.ipc.RemoteException;
 
 import com.google.common.annotations.VisibleForTesting;
+

hbase git commit: HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in CleanerChore) to branch-1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 de0dd9e82 -> 427c95e2e


HBASE-20352 [Chore] Backport HBASE-18309 (Support multi threads in 
CleanerChore) to branch-1

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/427c95e2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/427c95e2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/427c95e2

Branch: refs/heads/branch-1
Commit: 427c95e2e555e76329286d0a8596693bf567cbf7
Parents: de0dd9e
Author: Reid Chan 
Authored: Wed Apr 11 14:16:08 2018 +0800
Committer: Yu Li 
Committed: Wed Apr 11 14:17:51 2018 +0800

--
 .../org/apache/hadoop/hbase/master/HMaster.java |   4 +
 .../hbase/master/cleaner/CleanerChore.java  | 401 ++-
 .../hadoop/hbase/master/cleaner/LogCleaner.java | 181 +
 .../TestZooKeeperTableArchiveClient.java|   3 +
 .../hbase/master/cleaner/TestCleanerChore.java  |  99 -
 .../hbase/master/cleaner/TestHFileCleaner.java  |   1 +
 .../master/cleaner/TestHFileLinkCleaner.java|   1 +
 .../hbase/master/cleaner/TestLogsCleaner.java   |  57 +++
 8 files changed, 647 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/427c95e2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 6951098..85d5e23 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -97,6 +97,7 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
 import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
+import org.apache.hadoop.hbase.master.cleaner.CleanerChore;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationZKLockCleanerChore;
@@ -893,6 +894,7 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
(System.currentTimeMillis() - masterActiveTime) / 1000.0f));
 this.masterFinishedInitializationTime = System.currentTimeMillis();
 configurationManager.registerObserver(this.balancer);
+configurationManager.registerObserver(this.logCleaner);
 
 // Set master as 'initialized'.
 setInitialized(true);
@@ -1215,6 +1217,8 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
startProcedureExecutor();
 
+// Initial cleaner chore
+CleanerChore.initChorePool(conf);
// Start log cleaner thread
int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 
1000);
this.logCleaner =

http://git-wip-us.apache.org/repos/asf/hbase/blob/427c95e2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 743d9a4..50d8da3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.hbase.master.cleaner;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinTask;
+import java.util.concurrent.RecursiveTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -28,12 +32,16 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.FileStatusFilter;
+import org.apache.hadoop.ipc.RemoteException;
 
 import com.google.common.annotations.VisibleForTesting;
+impo

hbase git commit: HBASE-20159 Support using separate ZK quorums for client

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 78452113c -> b72e19e3b


HBASE-20159 Support using separate ZK quorums for client


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b72e19e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b72e19e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b72e19e3

Branch: refs/heads/branch-2
Commit: b72e19e3b989fa46c72d356703f5c0334b83fe76
Parents: 7845211
Author: Yu Li 
Authored: Thu Mar 29 02:37:26 2018 +0800
Committer: Yu Li 
Committed: Thu Mar 29 03:20:52 2018 +0800

--
 .../hbase/zookeeper/ReadOnlyZKClient.java   |   8 +-
 .../org/apache/hadoop/hbase/HConstants.java |  17 +-
 .../apache/hadoop/hbase/zookeeper/ZKConfig.java |  24 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  47 +++-
 .../hbase/master/zksyncer/ClientZKSyncer.java   | 241 +
 .../master/zksyncer/MasterAddressSyncer.java|  52 
 .../master/zksyncer/MetaLocationSyncer.java |  46 
 .../hbase/regionserver/HRegionServer.java   |  14 +
 .../regionserver/ReplicationSink.java   |   4 +
 .../client/TestSeparateClientZKCluster.java | 268 +++
 .../hbase/master/TestMasterNoCluster.java   |  37 +++
 .../hbase/zookeeper/MiniZooKeeperCluster.java   |   2 +-
 .../hadoop/hbase/zookeeper/ZKServerTool.java|   2 +-
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java   |   2 +-
 .../hadoop/hbase/zookeeper/ZKWatcher.java   |  37 ++-
 15 files changed, 781 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b72e19e3/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
index d2f4763..fc2d5f0 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
@@ -124,7 +124,13 @@ public final class ReadOnlyZKClient implements Closeable {
   }
 
   public ReadOnlyZKClient(Configuration conf) {
-this.connectString = ZKConfig.getZKQuorumServersString(conf);
+// We might use a different ZK for client access
+String clientZkQuorumServers = 
ZKConfig.getClientZKQuorumServersString(conf);
+if (clientZkQuorumServers != null) {
+  this.connectString = clientZkQuorumServers;
+} else {
+  this.connectString = ZKConfig.getZKQuorumServersString(conf);
+}
 this.sessionTimeoutMs = conf.getInt(ZK_SESSION_TIMEOUT, 
DEFAULT_ZK_SESSION_TIMEOUT);
 this.maxRetries = conf.getInt(RECOVERY_RETRY, DEFAULT_RECOVERY_RETRY);
 this.retryIntervalMs =

http://git-wip-us.apache.org/repos/asf/hbase/blob/b72e19e3/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 372d9b1..7ee31a5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -187,6 +187,19 @@ public final class HConstants {
   /** Name of ZooKeeper quorum configuration parameter. */
   public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
 
+  /** Name of ZooKeeper quorum configuration parameter for client to locate 
meta. */
+  public static final String CLIENT_ZOOKEEPER_QUORUM = 
"hbase.client.zookeeper.quorum";
+
+  /** Client port of ZooKeeper for client to locate meta */
+  public static final String CLIENT_ZOOKEEPER_CLIENT_PORT =
+  "hbase.client.zookeeper.property.clientPort";
+
+  /** Indicate whether the client ZK are observer nodes of the server ZK */
+  public static final String CLIENT_ZOOKEEPER_OBSERVER_MODE =
+  "hbase.client.zookeeper.observer.mode";
+  /** Assuming client zk not in observer mode and master need to synchronize 
information */
+  public static final boolean DEFAULT_CLIENT_ZOOKEEPER_OBSERVER_MODE = false;
+
   /** Common prefix of ZooKeeper configuration properties */
   public static final String ZK_CFG_PROPERTY_PREFIX =
   "hbase.zookeeper.property.";
@@ -205,7 +218,7 @@ public final class HConstants {
   ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
 
   /** Default client port that the zookeeper listens on */
-  public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181;
+  public static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 2181;
 
   /**
* Parameter name for the wait time for the recoverable zookeeper

hbase git commit: HBASE-20159 Support using separate ZK quorums for client

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 3b6199a27 -> 061a31fad


HBASE-20159 Support using separate ZK quorums for client


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/061a31fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/061a31fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/061a31fa

Branch: refs/heads/master
Commit: 061a31fad1654d9ded96d118e04c14860413fa25
Parents: 3b6199a
Author: Yu Li 
Authored: Thu Mar 29 02:37:26 2018 +0800
Committer: Yu Li 
Committed: Thu Mar 29 02:37:26 2018 +0800

--
 .../hbase/zookeeper/ReadOnlyZKClient.java   |   8 +-
 .../org/apache/hadoop/hbase/HConstants.java |  17 +-
 .../apache/hadoop/hbase/zookeeper/ZKConfig.java |  24 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  47 +++-
 .../hbase/master/zksyncer/ClientZKSyncer.java   | 241 +
 .../master/zksyncer/MasterAddressSyncer.java|  52 
 .../master/zksyncer/MetaLocationSyncer.java |  46 
 .../hbase/regionserver/HRegionServer.java   |  14 +
 .../regionserver/ReplicationSink.java   |   4 +
 .../client/TestSeparateClientZKCluster.java | 268 +++
 .../hbase/master/TestMasterNoCluster.java   |  37 +++
 .../hbase/zookeeper/MiniZooKeeperCluster.java   |   2 +-
 .../hadoop/hbase/zookeeper/ZKServerTool.java|   2 +-
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java   |   2 +-
 .../hadoop/hbase/zookeeper/ZKWatcher.java   |  37 ++-
 15 files changed, 781 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/061a31fa/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
index d2f4763..fc2d5f0 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java
@@ -124,7 +124,13 @@ public final class ReadOnlyZKClient implements Closeable {
   }
 
   public ReadOnlyZKClient(Configuration conf) {
-this.connectString = ZKConfig.getZKQuorumServersString(conf);
+// We might use a different ZK for client access
+String clientZkQuorumServers = 
ZKConfig.getClientZKQuorumServersString(conf);
+if (clientZkQuorumServers != null) {
+  this.connectString = clientZkQuorumServers;
+} else {
+  this.connectString = ZKConfig.getZKQuorumServersString(conf);
+}
 this.sessionTimeoutMs = conf.getInt(ZK_SESSION_TIMEOUT, 
DEFAULT_ZK_SESSION_TIMEOUT);
 this.maxRetries = conf.getInt(RECOVERY_RETRY, DEFAULT_RECOVERY_RETRY);
 this.retryIntervalMs =

http://git-wip-us.apache.org/repos/asf/hbase/blob/061a31fa/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 9a43e7c..9241682 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -183,6 +183,19 @@ public final class HConstants {
   /** Name of ZooKeeper quorum configuration parameter. */
   public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
 
+  /** Name of ZooKeeper quorum configuration parameter for client to locate 
meta. */
+  public static final String CLIENT_ZOOKEEPER_QUORUM = 
"hbase.client.zookeeper.quorum";
+
+  /** Client port of ZooKeeper for client to locate meta */
+  public static final String CLIENT_ZOOKEEPER_CLIENT_PORT =
+  "hbase.client.zookeeper.property.clientPort";
+
+  /** Indicate whether the client ZK are observer nodes of the server ZK */
+  public static final String CLIENT_ZOOKEEPER_OBSERVER_MODE =
+  "hbase.client.zookeeper.observer.mode";
+  /** Assuming client zk not in observer mode and master need to synchronize 
information */
+  public static final boolean DEFAULT_CLIENT_ZOOKEEPER_OBSERVER_MODE = false;
+
   /** Common prefix of ZooKeeper configuration properties */
   public static final String ZK_CFG_PROPERTY_PREFIX =
   "hbase.zookeeper.property.";
@@ -201,7 +214,7 @@ public final class HConstants {
   ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
 
   /** Default client port that the zookeeper listens on */
-  public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181;
+  public static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 2181;
 
   /** Parameter name for the root dir in ZK for this cluster */
   public stati

[2/2] hbase git commit: HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted

[liyu]

HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent 
write handler exhausted

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4f54a667
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4f54a667
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4f54a667

Branch: refs/heads/master
Commit: 4f54a66782e1621437a4c8eeea59aec37efa08c4
Parents: 9a26af3
Author: Chance Li 
Authored: Wed Mar 14 18:18:50 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:38:33 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +++-
 .../hadoop/hbase/regionserver/HStore.java   |  39 +++-
 .../hbase/regionserver/RSRpcServices.java   |   6 +
 .../apache/hadoop/hbase/regionserver/Store.java |   2 +
 .../throttle/StoreHotnessProtector.java | 196 +++
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 +
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 458 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4f54a667/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 0039a56..f74b5e0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,6 +89,7 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
+STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/4f54a667/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f071baf..0a94846 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -146,6 +146,7 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -674,6 +675,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
+  private final StoreHotnessProtector storeHotnessProtector;
+
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -794,6 +797,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
+
+this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
+
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -806,8 +812,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name as string and its encoded name.
-  LOG.debug("Instantiated " + this);
+  // Write out region name, its encoded name and storeHotnessProtector as 
string.
+  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
 }
 
 configurationManager = Optional.empty();
@@ -3180,9 +3186,31 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 if (!

[1/2] hbase git commit: Revert "HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted"

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 98ac4f12b -> 4f54a6678


Revert "HBASE-19389 Limit concurrency of put with dense (hundreds) columns to 
prevent write handler exhausted"

This reverts commit 98ac4f12b5ccec708fc03ddfb96935c8cd7304e1.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a26af37
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a26af37
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a26af37

Branch: refs/heads/master
Commit: 9a26af37b96332e0785f5ef4b19daa5dff3575cc
Parents: 98ac4f1
Author: Yu Li 
Authored: Wed Mar 14 18:38:07 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:38:07 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 -
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +---
 .../hadoop/hbase/regionserver/HStore.java   |  39 +---
 .../hbase/regionserver/RSRpcServices.java   |   6 -
 .../apache/hadoop/hbase/regionserver/Store.java |   2 -
 .../throttle/StoreHotnessProtector.java | 196 ---
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 -
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 11 insertions(+), 458 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a26af37/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index f74b5e0..0039a56 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,7 +89,6 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
-STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9a26af37/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 0a94846..f071baf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -146,7 +146,6 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
-import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -675,8 +674,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
-  private final StoreHotnessProtector storeHotnessProtector;
-
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -797,9 +794,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
-
-this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
-
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -812,8 +806,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name, its encoded name and storeHotnessProtector as 
string.
-  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
+  // Write out region name as string and its encoded name.
+  LOG.debug("Instantiated " + this);
 }
 
 configurationManager = Optional.em

[2/2] hbase git commit: HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted

[liyu]

HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent 
write handler exhausted

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/47b227b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/47b227b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/47b227b5

Branch: refs/heads/branch-2
Commit: 47b227b55b8104bfc59c639467f79ebb3cab7376
Parents: c8ecfc5
Author: Chance Li 
Authored: Wed Mar 14 18:18:50 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:36:58 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +++-
 .../hadoop/hbase/regionserver/HStore.java   |  39 +++-
 .../hbase/regionserver/RSRpcServices.java   |   6 +
 .../apache/hadoop/hbase/regionserver/Store.java |   2 +
 .../throttle/StoreHotnessProtector.java | 196 +++
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 +
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 458 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/47b227b5/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index adc7194..d826ca0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,6 +89,7 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
+STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/47b227b5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9f3d9bd..ba7ab5b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -145,6 +145,7 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -673,6 +674,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
+  private final StoreHotnessProtector storeHotnessProtector;
+
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -793,6 +796,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
+
+this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
+
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -805,8 +811,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name as string and its encoded name.
-  LOG.debug("Instantiated " + this);
+  // Write out region name, its encoded name and storeHotnessProtector as 
string.
+  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
 }
 
 configurationManager = Optional.empty();
@@ -3169,9 +3175,31 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 if 

[1/2] hbase git commit: Revert "HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted"

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 c6d89b494 -> 47b227b55


Revert "HBASE-19389 Limit concurrency of put with dense (hundreds) columns to 
prevent write handler exhausted"

This reverts commit c6d89b494c1b217e92b58e95c6da1934462edd33.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c8ecfc54
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c8ecfc54
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c8ecfc54

Branch: refs/heads/branch-2
Commit: c8ecfc54613663e855540c4927fab61a32263b99
Parents: c6d89b4
Author: Yu Li 
Authored: Wed Mar 14 18:35:22 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:35:22 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 -
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +---
 .../hadoop/hbase/regionserver/HStore.java   |  39 +---
 .../hbase/regionserver/RSRpcServices.java   |   6 -
 .../apache/hadoop/hbase/regionserver/Store.java |   2 -
 .../throttle/StoreHotnessProtector.java | 196 ---
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 -
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 11 insertions(+), 458 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c8ecfc54/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index d826ca0..adc7194 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,7 +89,6 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
-STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c8ecfc54/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index ba7ab5b..9f3d9bd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -145,7 +145,6 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
-import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -674,8 +673,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
-  private final StoreHotnessProtector storeHotnessProtector;
-
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -796,9 +793,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
-
-this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
-
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -811,8 +805,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name, its encoded name and storeHotnessProtector as 
string.
-  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
+  // Write out region name as string and its encoded name.
+  LOG.debug("Instantiated " + this);
 }
 
 configurationManager = Optiona

hbase git commit: HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 ac9035db1 -> c6d89b494


HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent 
write handler exhausted

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c6d89b49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c6d89b49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c6d89b49

Branch: refs/heads/branch-2
Commit: c6d89b494c1b217e92b58e95c6da1934462edd33
Parents: ac9035d
Author: Yu Li 
Authored: Wed Mar 14 18:18:50 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:33:39 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +++-
 .../hadoop/hbase/regionserver/HStore.java   |  39 +++-
 .../hbase/regionserver/RSRpcServices.java   |   6 +
 .../apache/hadoop/hbase/regionserver/Store.java |   2 +
 .../throttle/StoreHotnessProtector.java | 196 +++
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 +
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 458 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c6d89b49/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index adc7194..d826ca0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,6 +89,7 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
+STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6d89b49/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9f3d9bd..ba7ab5b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -145,6 +145,7 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -673,6 +674,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
+  private final StoreHotnessProtector storeHotnessProtector;
+
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -793,6 +796,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
+
+this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
+
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -805,8 +811,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name as string and its encoded name.
-  LOG.debug("Instantiated " + this);
+  // Write out region name, its encoded name and storeHotnessProtector as 
string.
+  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
 }
 
 configurationManager = Optional.empty();
@@ -3169,9 +3175,31 @@ public class HR

hbase git commit: HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent write handler exhausted

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 31978c31b -> 98ac4f12b


HBASE-19389 Limit concurrency of put with dense (hundreds) columns to prevent 
write handler exhausted

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/98ac4f12
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/98ac4f12
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/98ac4f12

Branch: refs/heads/master
Commit: 98ac4f12b5ccec708fc03ddfb96935c8cd7304e1
Parents: 31978c3
Author: Yu Li 
Authored: Wed Mar 14 18:18:50 2018 +0800
Committer: Yu Li 
Committed: Wed Mar 14 18:25:17 2018 +0800

--
 .../org/apache/hadoop/hbase/HConstants.java |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  86 +++-
 .../hadoop/hbase/regionserver/HStore.java   |  39 +++-
 .../hbase/regionserver/RSRpcServices.java   |   6 +
 .../apache/hadoop/hbase/regionserver/Store.java |   2 +
 .../throttle/StoreHotnessProtector.java | 196 +++
 .../apache/hadoop/hbase/io/TestHeapSize.java|   9 +
 .../throttle/TestStoreHotnessProtector.java | 130 
 8 files changed, 458 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/98ac4f12/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 0039a56..f74b5e0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -89,6 +89,7 @@ public final class HConstants {
 NOT_RUN,
 SUCCESS,
 BAD_FAMILY,
+STORE_TOO_BUSY,
 SANITY_CHECK_FAILURE,
 FAILURE
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/98ac4f12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f071baf..0a94846 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -146,6 +146,7 @@ import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import 
org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
 import 
org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
 import 
org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;
 import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
 import org.apache.hadoop.hbase.security.User;
@@ -674,6 +675,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final NavigableMap replicationScope = new TreeMap<>(
   Bytes.BYTES_COMPARATOR);
 
+  private final StoreHotnessProtector storeHotnessProtector;
+
   /**
* HRegion constructor. This constructor should only be used for testing and
* extensions.  Instances of HRegion should be instantiated with the
@@ -794,6 +797,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
 this.regionDurability = htd.getDurability() == Durability.USE_DEFAULT ?
 DEFAULT_DURABILITY : htd.getDurability();
+
+this.storeHotnessProtector = new StoreHotnessProtector(this, conf);
+
 if (rsServices != null) {
   this.rsAccounting = this.rsServices.getRegionServerAccounting();
   // don't initialize coprocessors if not running within a regionserver
@@ -806,8 +812,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.metricsRegion = null;
 }
 if (LOG.isDebugEnabled()) {
-  // Write out region name as string and its encoded name.
-  LOG.debug("Instantiated " + this);
+  // Write out region name, its encoded name and storeHotnessProtector as 
string.
+  LOG.debug("Instantiated " + this +"; "+ 
storeHotnessProtector.toString());
 }
 
 configurationManager = Optional.empty();
@@ -3180,9 +3186,31 @@ public class HRegio

hbase git commit: HBASE-19358 Improve the stability of splitting log when do fail over

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 a66e92310 -> 1e352f624


HBASE-19358 Improve the stability of splitting log when do fail over

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e352f62
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e352f62
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e352f62

Branch: refs/heads/branch-2
Commit: 1e352f6247bd516d85e3262796621ed5da31d298
Parents: a66e923
Author: Jingyun Tian 
Authored: Mon Jan 8 10:55:21 2018 +0800
Committer: Yu Li 
Committed: Mon Jan 8 10:57:46 2018 +0800

--
 .../apache/hadoop/hbase/wal/WALSplitter.java| 400 +--
 .../TestWALReplayBoundedLogWriterCreation.java  |  35 ++
 .../TestWALSplitBoundedLogWriterCreation.java   |  44 ++
 3 files changed, 358 insertions(+), 121 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1e352f62/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 4af782d..22f7e1a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -18,11 +18,6 @@
  */
 package org.apache.hadoop.hbase.wal;
 
-import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
-import 
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -31,6 +26,7 @@ import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
@@ -71,6 +67,7 @@ import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.log.HBaseMarkers;
 import org.apache.hadoop.hbase.master.SplitLogManager;
@@ -80,12 +77,6 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.LastSequenceId;
 import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
-import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -101,6 +92,16 @@ import org.apache.hadoop.ipc.RemoteException;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import 
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
 /**
  * This class is responsible for splitting up a bunch of regionserver commit 
log
  * files that are no longer being written to, into new files, one per region, 
for
@@ -140,6 +141,12 @@ public class WALSplitter {
   // the file being split currently
   private FileStatus fileBeingSplit;
 
+  // if we limit the number of writers opened for sinking recovered edits
+  private final boolean splitWriterCr

hbase git commit: HBASE-19358 Improve the stability of splitting log when do fail over

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 e866e837f -> 6faed49ad


HBASE-19358 Improve the stability of splitting log when do fail over

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6faed49a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6faed49a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6faed49a

Branch: refs/heads/branch-1
Commit: 6faed49ad1987e9b68ba22954b6b48c341136c95
Parents: e866e83
Author: Jingyun Tian 
Authored: Tue Jan 2 17:21:32 2018 +0800
Committer: Yu Li 
Committed: Tue Jan 2 17:25:49 2018 +0800

--
 .../apache/hadoop/hbase/wal/WALSplitter.java| 358 +--
 .../TestWALReplayBoundedLogWriterCreation.java  |  33 ++
 .../TestWALSplitBoundedLogWriterCreation.java   |  44 +++
 3 files changed, 330 insertions(+), 105 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6faed49a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 7c74649..0781562 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -25,6 +25,7 @@ import java.io.InterruptedIOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -163,6 +164,9 @@ public class WALSplitter {
 
   protected boolean distributedLogReplay;
 
+  private final boolean splitWriterCreationBounded;
+
+
   // Map encodedRegionName -> lastFlushedSequenceId
   protected Map lastFlushedSequenceIds = new 
ConcurrentHashMap();
 
@@ -182,6 +186,8 @@ public class WALSplitter {
   // the file being split currently
   private FileStatus fileBeingSplit;
 
+  public final static String SPLIT_WRITER_CREATION_BOUNDED = 
"hbase.split.writer.creation.bounded";
+
   @VisibleForTesting
   WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,
   FileSystem fs, LastSequenceId idChecker,
@@ -196,10 +202,10 @@ public class WALSplitter {
 this.csm = (BaseCoordinatedStateManager)csm;
 this.walFactory = factory;
 this.controller = new PipelineController();
-
+this.splitWriterCreationBounded = 
conf.getBoolean(SPLIT_WRITER_CREATION_BOUNDED, false);
 entryBuffers = new EntryBuffers(controller,
 this.conf.getInt("hbase.regionserver.hlog.splitlog.buffersize",
-128*1024*1024));
+128*1024*1024), splitWriterCreationBounded);
 
 // a larger minBatchSize may slow down recovery because replay writer has 
to wait for
 // enough edits before replaying them
@@ -214,7 +220,12 @@ public class WALSplitter {
 LOG.info("ZooKeeperWatcher is passed in as NULL so disable 
distrubitedLogRepaly.");
   }
   this.distributedLogReplay = false;
-  outputSink = new LogRecoveredEditsOutputSink(controller, entryBuffers, 
numWriterThreads);
+  if(splitWriterCreationBounded){
+outputSink = new BoundedLogWriterCreationOutputSink(controller,
+entryBuffers, numWriterThreads);
+  }else {
+outputSink = new LogRecoveredEditsOutputSink(controller, entryBuffers, 
numWriterThreads);
+  }
 }
 
   }
@@ -925,11 +936,19 @@ public class WALSplitter {
 Set currentlyWriting = new TreeSet(Bytes.BYTES_COMPARATOR);
 
 long totalBuffered = 0;
-long maxHeapUsage;
+final long maxHeapUsage;
+boolean splitWriterCreationBounded;
+
 
 public EntryBuffers(PipelineController controller, long maxHeapUsage) {
+  this(controller, maxHeapUsage, false);
+}
+
+public EntryBuffers(PipelineController controller, long maxHeapUsage,
+boolean splitWriterCreationBounded) {
   this.controller = controller;
   this.maxHeapUsage = maxHeapUsage;
+  this.splitWriterCreationBounded = splitWriterCreationBounded;
 }
 
 /**
@@ -969,6 +988,14 @@ public class WALSplitter {
  * @return RegionEntryBuffer a buffer of edits to be written or replayed.
  */
 synchronized RegionEntryBuffer getChunkToWrite() {
+  // The core part of limiting opening writers is it doesn't return chunk 
only if the heap size
+  // is over maxHeapUsage. Thus it doesn't need to create a writer for 
each region
+  // during splitting. It will flush all the logs in the buffer after 
splitting through a
+  // threadpool, which means the number of writers it created is under 
control
+  if(splitWriterCreationBounded && totalBuffered < maxHeapU

hbase git commit: HBASE-19358 Improve the stability of splitting log when do fail over

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 4e9f4abb1 -> f6f57d38f


HBASE-19358 Improve the stability of splitting log when do fail over

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f6f57d38
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f6f57d38
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f6f57d38

Branch: refs/heads/master
Commit: f6f57d38f750c34e2353f605f815c019c8ba3f88
Parents: 4e9f4ab
Author: Jingyun Tian 
Authored: Tue Jan 2 19:17:34 2018 +0800
Committer: Yu Li 
Committed: Wed Jan 3 04:11:40 2018 +0800

--
 .../apache/hadoop/hbase/wal/WALSplitter.java| 400 +--
 .../TestWALReplayBoundedLogWriterCreation.java  |  35 ++
 .../TestWALSplitBoundedLogWriterCreation.java   |  44 ++
 3 files changed, 358 insertions(+), 121 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f6f57d38/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 817e925..328390e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -18,11 +18,6 @@
  */
 package org.apache.hadoop.hbase.wal;
 
-import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
-import 
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -30,6 +25,7 @@ import java.io.InterruptedIOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -68,6 +64,7 @@ import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.log.HBaseMarkers;
 import org.apache.hadoop.hbase.master.SplitLogManager;
@@ -77,12 +74,6 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.LastSequenceId;
 import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
-import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
-import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -99,6 +90,16 @@ import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import 
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
+
 /**
  * This class is responsible for splitting up a bunch of regionserver commit 
log
  * files that are no longer being written to, into new files, one per region, 
for
@@ -138,6 +139,12 @@ public class WALSplitter {
   // the file being split currently
   private FileStatus fileBeingSplit;
 
+  // if we limit the number of writers opened for sinking recovered edits
+  private final boolean splitWriterCreationBounded;
+
+  public final stati

hbase git commit: HBASE-18309 Support multi threads in CleanerChore

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 66e650c39 -> 2838cf3e0


HBASE-18309 Support multi threads in CleanerChore

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2838cf3e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2838cf3e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2838cf3e

Branch: refs/heads/branch-2
Commit: 2838cf3e0506d9be8c001ebbadc0c3acf68d762c
Parents: 66e650c
Author: Reid Chan 
Authored: Thu Nov 23 11:21:12 2017 +0800
Committer: Yu Li 
Committed: Fri Nov 24 13:36:55 2017 +0800

--
 .../org/apache/hadoop/hbase/master/HMaster.java |   1 +
 .../hbase/master/cleaner/CleanerChore.java  | 335 +--
 .../hbase/master/cleaner/HFileCleaner.java  |   8 +-
 .../hadoop/hbase/master/cleaner/LogCleaner.java | 168 +-
 .../hbase/master/cleaner/TestCleanerChore.java  |  61 +++-
 .../hbase/master/cleaner/TestLogsCleaner.java   |  51 +++
 6 files changed, 514 insertions(+), 110 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2838cf3e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index fcaf55f..cde3581 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -908,6 +908,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 this.masterFinishedInitializationTime = System.currentTimeMillis();
 configurationManager.registerObserver(this.balancer);
 configurationManager.registerObserver(this.hfileCleaner);
+configurationManager.registerObserver(this.logCleaner);
 
 // Set master as 'initialized'.
 setInitialized(true);

http://git-wip-us.apache.org/repos/asf/hbase/blob/2838cf3e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index b8ca1ec..582df84 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -17,18 +17,22 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import 
org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Predicate;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RecursiveTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -46,16 +50,33 @@ import org.apache.hadoop.ipc.RemoteException;
  * Abstract Cleaner that uses a chain of delegates to clean a directory of 
files
  * @param  Cleaner delegate class that is dynamically loaded from 
configuration
  */
-public abstract class CleanerChore extends 
ScheduledChore {
+public abstract class CleanerChore extends 
ScheduledChore
+implements ConfigurationObserver {
 
   private static final Log LOG = 
LogFactory.getLog(CleanerChore.class.getName());
+  private static final int AVAIL_PROCESSORS = 
Runtime.getRuntime().availableProcessors();
+
+  /**
+   * If it is an integer and >= 1, it would be the size;
+   * if 0.0 < size <= 1.0, size would be available processors * size.
+   * Pay attention that 1.0 is different from 1, former indicates it will use 
100% of cores,
+   * while latter will use only 1 thread for chore to scan dir.
+   */
+  public static final String CHORE_POOL_SIZE = 
"hbase.cleaner.scan.dir.concurrent.size";
+  private static final String DEFAULT_CHORE_POOL_SIZE = "0.5";
+
+  // It may be waste resources for each cleaner chore own its pool,
+  // so let's make pool for all cleaner chores.
+  private static volatile ForkJoinPool chorePool;
+  private static volatile int chorePoolSize;
 
   protected final F

hbase git commit: HBASE-18309 Support multi threads in CleanerChore

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master cdc2bb17f -> 2442cbb6a


HBASE-18309 Support multi threads in CleanerChore

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2442cbb6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2442cbb6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2442cbb6

Branch: refs/heads/master
Commit: 2442cbb6ab5b9f1729b74361dd2bbb066d5910bd
Parents: cdc2bb1
Author: Reid Chan 
Authored: Thu Nov 23 11:21:12 2017 +0800
Committer: Yu Li 
Committed: Thu Nov 23 11:23:26 2017 +0800

--
 .../org/apache/hadoop/hbase/master/HMaster.java |   1 +
 .../hbase/master/cleaner/CleanerChore.java  | 335 +--
 .../hbase/master/cleaner/HFileCleaner.java  |   8 +-
 .../hadoop/hbase/master/cleaner/LogCleaner.java | 168 +-
 .../hbase/master/cleaner/TestCleanerChore.java  |  61 +++-
 .../hbase/master/cleaner/TestLogsCleaner.java   |  51 +++
 6 files changed, 514 insertions(+), 110 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2442cbb6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 97982b9..cfbddfc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -908,6 +908,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 this.masterFinishedInitializationTime = System.currentTimeMillis();
 configurationManager.registerObserver(this.balancer);
 configurationManager.registerObserver(this.hfileCleaner);
+configurationManager.registerObserver(this.logCleaner);
 
 // Set master as 'initialized'.
 setInitialized(true);

http://git-wip-us.apache.org/repos/asf/hbase/blob/2442cbb6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index b8ca1ec..582df84 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -17,18 +17,22 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import 
org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.shaded.com.google.common.base.Predicate;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.Iterables;
 import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RecursiveTask;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
@@ -46,16 +50,33 @@ import org.apache.hadoop.ipc.RemoteException;
  * Abstract Cleaner that uses a chain of delegates to clean a directory of 
files
  * @param  Cleaner delegate class that is dynamically loaded from 
configuration
  */
-public abstract class CleanerChore extends 
ScheduledChore {
+public abstract class CleanerChore extends 
ScheduledChore
+implements ConfigurationObserver {
 
   private static final Log LOG = 
LogFactory.getLog(CleanerChore.class.getName());
+  private static final int AVAIL_PROCESSORS = 
Runtime.getRuntime().availableProcessors();
+
+  /**
+   * If it is an integer and >= 1, it would be the size;
+   * if 0.0 < size <= 1.0, size would be available processors * size.
+   * Pay attention that 1.0 is different from 1, former indicates it will use 
100% of cores,
+   * while latter will use only 1 thread for chore to scan dir.
+   */
+  public static final String CHORE_POOL_SIZE = 
"hbase.cleaner.scan.dir.concurrent.size";
+  private static final String DEFAULT_CHORE_POOL_SIZE = "0.5";
+
+  // It may be waste resources for each cleaner chore own its pool,
+  // so let's make pool for all cleaner chores.
+  private static volatile ForkJoinPool chorePool;
+  private static volatile int chorePoolSize;
 
   protected final FileS

hbase git commit: HBASE-19016 Coordinate storage policy property name for table schema and bulkload

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 58b0585d6 -> cf550c2be


HBASE-19016 Coordinate storage policy property name for table schema and 
bulkload


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf550c2b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf550c2b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf550c2b

Branch: refs/heads/branch-2
Commit: cf550c2be123c91b44b110b5c5ef5060008cc0a7
Parents: 58b0585
Author: Yu Li 
Authored: Tue Oct 17 10:24:07 2017 +0800
Committer: Yu Li 
Committed: Tue Oct 17 10:25:27 2017 +0800

--
 .../java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cf550c2b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
--
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 20b2d42..fdbe6c0 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -172,7 +172,7 @@ public class HFileOutputFormat2
   static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY =
   "hbase.mapreduce.use.multi.table.hfileoutputformat";
 
-  public static final String STORAGE_POLICY_PROPERTY = 
"hbase.hstore.storagepolicy";
+  public static final String STORAGE_POLICY_PROPERTY = 
HStore.BLOCK_STORAGE_POLICY_KEY;
   public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = 
STORAGE_POLICY_PROPERTY + ".";
 
   @Override

hbase git commit: HBASE-19016 Coordinate storage policy property name for table schema and bulkload

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 51489b208 -> be4570fc8


HBASE-19016 Coordinate storage policy property name for table schema and 
bulkload


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/be4570fc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/be4570fc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/be4570fc

Branch: refs/heads/master
Commit: be4570fc85ad51357fdb9eed3f402a59314ad41d
Parents: 51489b20
Author: Yu Li 
Authored: Tue Oct 17 10:24:07 2017 +0800
Committer: Yu Li 
Committed: Tue Oct 17 10:24:07 2017 +0800

--
 .../java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/be4570fc/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
--
diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 20b2d42..fdbe6c0 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -172,7 +172,7 @@ public class HFileOutputFormat2
   static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY =
   "hbase.mapreduce.use.multi.table.hfileoutputformat";
 
-  public static final String STORAGE_POLICY_PROPERTY = 
"hbase.hstore.storagepolicy";
+  public static final String STORAGE_POLICY_PROPERTY = 
HStore.BLOCK_STORAGE_POLICY_KEY;
   public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = 
STORAGE_POLICY_PROPERTY + ".";
 
   @Override

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 2598d0087 -> 68cd18176


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/68cd1817
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/68cd1817
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/68cd1817

Branch: refs/heads/branch-1.1
Commit: 68cd18176e197c08294af622b849f2e7f9ea1fc7
Parents: 2598d00
Author: Yu Li 
Authored: Fri Sep 8 11:40:26 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:53:53 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/68cd1817/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index 49c095d..8c3c212 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -1676,7 +1676,7 @@ class AsyncProcess {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
   + max + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", 
tableName=" + tableName));
   if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
 logDetailsOfUndoneTasks(currentInProgress);
   }

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 699b3e7b1 -> 025c3f47d


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/025c3f47
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/025c3f47
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/025c3f47

Branch: refs/heads/branch-1.3
Commit: 025c3f47da5a297838a00e673b2bc6f4f4ff1e16
Parents: 699b3e7
Author: Yu Li 
Authored: Fri Sep 8 11:40:26 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:54:34 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/025c3f47/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index d5f99f1..12fbb47 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -1819,7 +1819,7 @@ class AsyncProcess {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
   + max + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", 
tableName=" + tableName));
   if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
 logDetailsOfUndoneTasks(currentInProgress);
   }

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 316dd45cc -> da25def91


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da25def9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da25def9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da25def9

Branch: refs/heads/branch-1.2
Commit: da25def91b517ecc279444d8ab4dd943c117f61c
Parents: 316dd45
Author: Yu Li 
Authored: Fri Sep 8 11:40:26 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:54:19 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/da25def9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index 0f1fd58..d49d474 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -1747,7 +1747,7 @@ class AsyncProcess {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
   + max + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", 
tableName=" + tableName));
   if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
 logDetailsOfUndoneTasks(currentInProgress);
   }

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.4 b87a8d97c -> a1d0bb97a


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1d0bb97
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1d0bb97
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1d0bb97

Branch: refs/heads/branch-1.4
Commit: a1d0bb97ac0a3384dab14e343c4394f09ac09eee
Parents: b87a8d9
Author: Yu Li 
Authored: Fri Sep 8 11:40:26 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:55:46 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a1d0bb97/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index 097206c..44a7656 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -1857,7 +1857,7 @@ class AsyncProcess {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
   + max + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", 
tableName=" + tableName));
   if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
 logDetailsOfUndoneTasks(currentInProgress);
   }

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 00d9e3847 -> 52c809c35


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52c809c3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52c809c3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52c809c3

Branch: refs/heads/branch-2
Commit: 52c809c35e2abaa9e9b21b2a5fb775b2edbcf0f0
Parents: 00d9e38
Author: Yu Li 
Authored: Fri Sep 8 11:38:00 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:55:11 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/52c809c3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index f2d8d74..5e0da59 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -448,7 +448,7 @@ class AsyncProcess {
   LOG.info("#" + id + (max < 0 ? ", waiting for any free slot"
   : ", waiting for some tasks to finish. Expected max="
   + max) + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", tableName=" + 
tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", tableName=" + 
tableName));
 };
   }
 

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 34d648808 -> b56760c73


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b56760c7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b56760c7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b56760c7

Branch: refs/heads/master
Commit: b56760c732f4f72fcc3734cb0133262198d0889c
Parents: 34d6488
Author: Yu Li 
Authored: Fri Sep 8 11:38:00 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:54:51 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b56760c7/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index f2d8d74..5e0da59 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -448,7 +448,7 @@ class AsyncProcess {
   LOG.info("#" + id + (max < 0 ? ", waiting for any free slot"
   : ", waiting for some tasks to finish. Expected max="
   + max) + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", tableName=" + 
tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", tableName=" + 
tableName));
 };
   }
 

hbase git commit: HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 ead106324 -> 539fce344


HBASE-15931 Add log for long-running tasks in AsyncProcess (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/539fce34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/539fce34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/539fce34

Branch: refs/heads/branch-1
Commit: 539fce344048514606dd0c350985875774ec5e95
Parents: ead1063
Author: Yu Li 
Authored: Fri Sep 8 11:40:26 2017 +0800
Committer: Yu Li 
Committed: Tue Sep 19 09:53:14 2017 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/539fce34/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index 097206c..44a7656 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -1857,7 +1857,7 @@ class AsyncProcess {
   lastLog = now;
   LOG.info("#" + id + ", waiting for some tasks to finish. Expected 
max="
   + max + ", tasksInProgress=" + currentInProgress +
-  " hasError=" + hasError() + tableName == null ? "" : ", 
tableName=" + tableName);
+  " hasError=" + hasError() + (tableName == null ? "" : ", 
tableName=" + tableName));
   if (currentInProgress <= thresholdToLogUndoneTaskDetails) {
 logDetailsOfUndoneTasks(currentInProgress);
   }

hbase git commit: HBASE-18469 Correct RegionServer metric of totalRequestCount

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 d72ba890d -> a03c2036e


HBASE-18469 Correct RegionServer metric of totalRequestCount


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a03c2036
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a03c2036
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a03c2036

Branch: refs/heads/branch-1
Commit: a03c2036e9433800db7a9d6cad632a210f5ede35
Parents: d72ba89
Author: Yu Li 
Authored: Fri Aug 11 14:13:18 2017 +0800
Committer: Yu Li 
Committed: Fri Aug 11 14:53:23 2017 +0800

--
 .../regionserver/MetricsRegionServerSource.java |  3 ++
 .../MetricsRegionServerWrapper.java |  2 +
 .../MetricsRegionServerSourceImpl.java  |  2 +
 .../MetricsRegionServerWrapperImpl.java |  5 +++
 .../hbase/regionserver/RSRpcServices.java   | 15 +--
 .../MetricsRegionServerWrapperStub.java |  5 +++
 .../regionserver/TestMetricsRegionServer.java   |  4 ++
 .../regionserver/TestRegionServerMetrics.java   | 41 ++--
 8 files changed, 71 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 06bdee6..3ac678e 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends 
BaseSource, JvmPauseMonitorSo
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =
   "Total number of requests this RegionServer has answered.";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+  "Total number of region requests this RegionServer has answered, count 
by row-level action";
   String READ_REQUEST_COUNT = "readRequestCount";
   String READ_REQUEST_COUNT_DESC =
   "Number of read requests this region server has answered.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 7232063..7d7f66d 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -367,4 +367,6 @@ public interface MetricsRegionServerWrapper {
   long getDeleteFamilyBloomHitCount();
 
   long getTrailerHitCount();
+
+  long getTotalRowActionRequestCount();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
--
diff --git 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9b7f80a..cec122b 100644
--- 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
   .addGauge(Interns.info(AVERAGE_REGION_SIZE, 
AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
   .addCounter(Interns.info(TOTAL_REQUEST_COUNT, 
TOTAL_REQUEST_COUNT_DESC),
   rsWrap.getTotalRequestCount())
+  .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, 
TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+  rsWrap.getTotalRowActionRequestCount())
   .addCounter(Interns.info(READ_REQUEST_COUNT, 
READ_REQUEST_COUNT_DESC),
   rsWrap.getReadRequestsCount())
   .addCounter(Interns.info(WRITE_REQUEST_COUNT, 
WRITE_REQUEST_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/a03c2036/hbase-se

hbase git commit: HBASE-18469 Correct RegionServer metric of totalRequestCount

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 8da77b414 -> 679f34e88


HBASE-18469 Correct RegionServer metric of totalRequestCount


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/679f34e8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/679f34e8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/679f34e8

Branch: refs/heads/master
Commit: 679f34e88108d870b79f3175b195e8e7d803c331
Parents: 8da77b4
Author: Yu Li 
Authored: Fri Aug 11 14:13:18 2017 +0800
Committer: Yu Li 
Committed: Fri Aug 11 14:13:18 2017 +0800

--
 .../regionserver/MetricsRegionServerSource.java |  3 ++
 .../MetricsRegionServerWrapper.java |  2 +
 .../MetricsRegionServerSourceImpl.java  |  2 +
 .../MetricsRegionServerWrapperImpl.java |  5 +++
 .../hbase/regionserver/RSRpcServices.java   | 14 ++-
 .../MetricsRegionServerWrapperStub.java |  5 +++
 .../regionserver/TestMetricsRegionServer.java   |  4 ++
 .../regionserver/TestRegionServerMetrics.java   | 40 +++-
 8 files changed, 71 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/679f34e8/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index df522d3..9656894 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends 
BaseSource, JvmPauseMonitorSo
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =
   "Total number of requests this RegionServer has answered.";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+  "Total number of region requests this RegionServer has answered, count 
by row-level action";
   String READ_REQUEST_COUNT = "readRequestCount";
   String READ_REQUEST_COUNT_DESC =
   "Number of read requests this region server has answered.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/679f34e8/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 0aa625c..ccb9de2 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -457,4 +457,6 @@ public interface MetricsRegionServerWrapper {
   long getDeleteFamilyBloomHitCount();
 
   long getTrailerHitCount();
+
+  long getTotalRowActionRequestCount();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/679f34e8/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
--
diff --git 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 94b21bc..e69e17c 100644
--- 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
   .addGauge(Interns.info(AVERAGE_REGION_SIZE, 
AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
   .addCounter(Interns.info(TOTAL_REQUEST_COUNT, 
TOTAL_REQUEST_COUNT_DESC),
   rsWrap.getTotalRequestCount())
+  .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, 
TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+  rsWrap.getTotalRowActionRequestCount())
   .addCounter(Interns.info(READ_REQUEST_COUNT, 
READ_REQUEST_COUNT_DESC),
   rsWrap.getReadRequestsCount())
   .addCounter(Interns.info(FILTERED_READ_REQUEST_COUNT, 
FILTERED_READ_REQUEST_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/679f3

hbase git commit: HBASE-18469 Correct RegionServer metric of totalRequestCount

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 ee15c2c29 -> 56a4fedda


HBASE-18469 Correct RegionServer metric of totalRequestCount


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/56a4fedd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/56a4fedd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/56a4fedd

Branch: refs/heads/branch-2
Commit: 56a4fedda20b91e3ea36d4078ee9463e13faec74
Parents: ee15c2c
Author: Yu Li 
Authored: Fri Aug 11 14:13:18 2017 +0800
Committer: Yu Li 
Committed: Fri Aug 11 14:15:03 2017 +0800

--
 .../regionserver/MetricsRegionServerSource.java |  3 ++
 .../MetricsRegionServerWrapper.java |  2 +
 .../MetricsRegionServerSourceImpl.java  |  2 +
 .../MetricsRegionServerWrapperImpl.java |  5 +++
 .../hbase/regionserver/RSRpcServices.java   | 14 ++-
 .../MetricsRegionServerWrapperStub.java |  5 +++
 .../regionserver/TestMetricsRegionServer.java   |  4 ++
 .../regionserver/TestRegionServerMetrics.java   | 40 +++-
 8 files changed, 71 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/56a4fedd/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index df522d3..9656894 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends 
BaseSource, JvmPauseMonitorSo
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =
   "Total number of requests this RegionServer has answered.";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+  "Total number of region requests this RegionServer has answered, count 
by row-level action";
   String READ_REQUEST_COUNT = "readRequestCount";
   String READ_REQUEST_COUNT_DESC =
   "Number of read requests this region server has answered.";

http://git-wip-us.apache.org/repos/asf/hbase/blob/56a4fedd/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 0aa625c..ccb9de2 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -457,4 +457,6 @@ public interface MetricsRegionServerWrapper {
   long getDeleteFamilyBloomHitCount();
 
   long getTrailerHitCount();
+
+  long getTotalRowActionRequestCount();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/56a4fedd/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
--
diff --git 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 94b21bc..e69e17c 100644
--- 
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ 
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
   .addGauge(Interns.info(AVERAGE_REGION_SIZE, 
AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
   .addCounter(Interns.info(TOTAL_REQUEST_COUNT, 
TOTAL_REQUEST_COUNT_DESC),
   rsWrap.getTotalRequestCount())
+  .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, 
TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+  rsWrap.getTotalRowActionRequestCount())
   .addCounter(Interns.info(READ_REQUEST_COUNT, 
READ_REQUEST_COUNT_DESC),
   rsWrap.getReadRequestsCount())
   .addCounter(Interns.info(FILTERED_READ_REQUEST_COUNT, 
FILTERED_READ_REQUEST_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/5

hbase git commit: HBASE-18441 ZookeeperWatcher#interruptedException should throw exception (addendum)

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 145f7d8a4 -> 5726263fb


HBASE-18441 ZookeeperWatcher#interruptedException should throw exception 
(addendum)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5726263f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5726263f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5726263f

Branch: refs/heads/branch-1
Commit: 5726263fb287767b2ef9024560899761f4ef025e
Parents: 145f7d8
Author: Yu Li 
Authored: Wed Jul 26 15:48:34 2017 +0800
Committer: Yu Li 
Committed: Wed Jul 26 15:48:34 2017 +0800

--
 .../test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5726263f/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
index 30a07e1..9990cd1 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
@@ -107,9 +107,9 @@ public class TestZKUtil {
   }
 };
 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
-.getChildren(zkw.znodePaths.baseZNode, null);
+.getChildren(zkw.assignmentZNode, null);
 try {
-  ZKUtil.listChildrenNoWatch(zkw, zkw.znodePaths.baseZNode);
+  ZKUtil.listChildrenNoWatch(zkw, zkw.assignmentZNode);
 } catch (KeeperException.SystemErrorException e) {
   // expected
   return;

hbase git commit: HBASE-18441 ZookeeperWatcher#interruptedException should throw exception

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 a4e1af1fc -> 145f7d8a4


HBASE-18441 ZookeeperWatcher#interruptedException should throw exception


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/145f7d8a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/145f7d8a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/145f7d8a

Branch: refs/heads/branch-1
Commit: 145f7d8a4d63df139e48fcd4b1691f5cf988333b
Parents: a4e1af1
Author: Yu Li 
Authored: Wed Jul 26 15:19:11 2017 +0800
Committer: Yu Li 
Committed: Wed Jul 26 15:21:05 2017 +0800

--
 .../hbase/zookeeper/ZooKeeperWatcher.java   | 32 
 .../hadoop/hbase/zookeeper/TestZKUtil.java  | 22 ++
 2 files changed, 41 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/145f7d8a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index f333cd5..e9e38c8 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -257,7 +257,7 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 } catch(KeeperException.NoNodeException nne) {
   return;
 } catch(InterruptedException ie) {
-  interruptedException(ie);
+  interruptedExceptionNoThrow(ie, false);
 } catch (IOException|KeeperException e) {
   LOG.warn("Received exception while checking and setting zookeeper ACLs", 
e);
 }
@@ -737,20 +737,26 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 
   /**
* Handles InterruptedExceptions in client calls.
-   * 
-   * This may be temporary but for now this gives one place to deal with these.
-   * 
-   * TODO: Currently, this method does nothing.
-   *   Is this ever expected to happen?  Do we abort or can we let it run?
-   *   Maybe this should be logged as WARN?  It shouldn't happen?
-   * 
-   * @param ie
+   * @param ie the InterruptedException instance thrown
+   * @throws KeeperException the exception to throw, transformed from the 
InterruptedException
+   */
+  public void interruptedException(InterruptedException ie) throws 
KeeperException {
+interruptedExceptionNoThrow(ie, true);
+// Throw a system error exception to let upper level handle it
+throw new KeeperException.SystemErrorException();
+  }
+
+  /**
+   * Log the InterruptedException and interrupt current thread
+   * @param ie The IterruptedException to log
+   * @param throwLater Whether we will throw the exception latter
*/
-  public void interruptedException(InterruptedException ie) {
-LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
-// At least preserver interrupt.
+  public void interruptedExceptionNoThrow(InterruptedException ie, boolean 
throwLater) {
+LOG.debug(prefix("Received InterruptedException, will interrupt current 
thread"
++ (throwLater ? " and rethrow a SystemErrorException" : "")),
+  ie);
+// At least preserve interrupt.
 Thread.currentThread().interrupt();
-// no-op
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/145f7d8a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
index 02d002a..30a07e1 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.security.Superusers;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooDefs.Perms;
 import org.apache.zookeeper.data.ACL;
@@ -35,6 +36,7 @@ import org.apache.zookeeper.data.Id;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 /**
  *
@@ -94,4 +96,24 @@ public class TestZKUtil {
 Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", 
"user5";
 Assert.assertTrue(aclList.conta

hbase git commit: HBASE-18441 ZookeeperWatcher#interruptedException should throw exception

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 7517f9326 -> 3be51a85b


HBASE-18441 ZookeeperWatcher#interruptedException should throw exception


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3be51a85
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3be51a85
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3be51a85

Branch: refs/heads/branch-2
Commit: 3be51a85ba4c1636bdebadef2ce1d1eda53098fa
Parents: 7517f93
Author: Yu Li 
Authored: Wed Jul 26 15:19:11 2017 +0800
Committer: Yu Li 
Committed: Wed Jul 26 15:21:25 2017 +0800

--
 .../hbase/zookeeper/ZooKeeperWatcher.java   | 32 
 .../hadoop/hbase/zookeeper/TestZKUtil.java  | 22 ++
 2 files changed, 41 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3be51a85/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index f18b8ba..6bec352 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -207,7 +207,7 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 } catch(KeeperException.NoNodeException nne) {
   return;
 } catch(InterruptedException ie) {
-  interruptedException(ie);
+  interruptedExceptionNoThrow(ie, false);
 } catch (IOException|KeeperException e) {
   LOG.warn("Received exception while checking and setting zookeeper ACLs", 
e);
 }
@@ -587,20 +587,26 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 
   /**
* Handles InterruptedExceptions in client calls.
-   * 
-   * This may be temporary but for now this gives one place to deal with these.
-   * 
-   * TODO: Currently, this method does nothing.
-   *   Is this ever expected to happen?  Do we abort or can we let it run?
-   *   Maybe this should be logged as WARN?  It shouldn't happen?
-   * 
-   * @param ie
+   * @param ie the InterruptedException instance thrown
+   * @throws KeeperException the exception to throw, transformed from the 
InterruptedException
+   */
+  public void interruptedException(InterruptedException ie) throws 
KeeperException {
+interruptedExceptionNoThrow(ie, true);
+// Throw a system error exception to let upper level handle it
+throw new KeeperException.SystemErrorException();
+  }
+
+  /**
+   * Log the InterruptedException and interrupt current thread
+   * @param ie The IterruptedException to log
+   * @param throwLater Whether we will throw the exception latter
*/
-  public void interruptedException(InterruptedException ie) {
-LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
-// At least preserver interrupt.
+  public void interruptedExceptionNoThrow(InterruptedException ie, boolean 
throwLater) {
+LOG.debug(prefix("Received InterruptedException, will interrupt current 
thread"
++ (throwLater ? " and rethrow a SystemErrorException" : "")),
+  ie);
+// At least preserve interrupt.
 Thread.currentThread().interrupt();
-// no-op
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/3be51a85/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
index 0e1ab92..53dcdbc 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.security.Superusers;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooDefs.Perms;
 import org.apache.zookeeper.data.ACL;
@@ -34,6 +35,7 @@ import org.apache.zookeeper.data.Id;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 /**
  *
@@ -93,4 +95,24 @@ public class TestZKUtil {
 Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", 
"user5";
 Assert.assertTrue(aclList.conta

hbase git commit: HBASE-18441 ZookeeperWatcher#interruptedException should throw exception

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 421029d0c -> 1e49decfa


HBASE-18441 ZookeeperWatcher#interruptedException should throw exception


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e49decf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e49decf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e49decf

Branch: refs/heads/master
Commit: 1e49decfa2bc72c039ed559055d41b1c2c2b3a61
Parents: 421029d
Author: Yu Li 
Authored: Wed Jul 26 15:19:11 2017 +0800
Committer: Yu Li 
Committed: Wed Jul 26 15:19:11 2017 +0800

--
 .../hbase/zookeeper/ZooKeeperWatcher.java   | 32 
 .../hadoop/hbase/zookeeper/TestZKUtil.java  | 22 ++
 2 files changed, 41 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1e49decf/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index f18b8ba..6bec352 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -207,7 +207,7 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 } catch(KeeperException.NoNodeException nne) {
   return;
 } catch(InterruptedException ie) {
-  interruptedException(ie);
+  interruptedExceptionNoThrow(ie, false);
 } catch (IOException|KeeperException e) {
   LOG.warn("Received exception while checking and setting zookeeper ACLs", 
e);
 }
@@ -587,20 +587,26 @@ public class ZooKeeperWatcher implements Watcher, 
Abortable, Closeable {
 
   /**
* Handles InterruptedExceptions in client calls.
-   * 
-   * This may be temporary but for now this gives one place to deal with these.
-   * 
-   * TODO: Currently, this method does nothing.
-   *   Is this ever expected to happen?  Do we abort or can we let it run?
-   *   Maybe this should be logged as WARN?  It shouldn't happen?
-   * 
-   * @param ie
+   * @param ie the InterruptedException instance thrown
+   * @throws KeeperException the exception to throw, transformed from the 
InterruptedException
+   */
+  public void interruptedException(InterruptedException ie) throws 
KeeperException {
+interruptedExceptionNoThrow(ie, true);
+// Throw a system error exception to let upper level handle it
+throw new KeeperException.SystemErrorException();
+  }
+
+  /**
+   * Log the InterruptedException and interrupt current thread
+   * @param ie The IterruptedException to log
+   * @param throwLater Whether we will throw the exception latter
*/
-  public void interruptedException(InterruptedException ie) {
-LOG.debug(prefix("Received InterruptedException, doing nothing here"), ie);
-// At least preserver interrupt.
+  public void interruptedExceptionNoThrow(InterruptedException ie, boolean 
throwLater) {
+LOG.debug(prefix("Received InterruptedException, will interrupt current 
thread"
++ (throwLater ? " and rethrow a SystemErrorException" : "")),
+  ie);
+// At least preserve interrupt.
 Thread.currentThread().interrupt();
-// no-op
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/1e49decf/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
index 0e1ab92..53dcdbc 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.security.Superusers;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooDefs.Perms;
 import org.apache.zookeeper.data.ACL;
@@ -34,6 +35,7 @@ import org.apache.zookeeper.data.Id;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 /**
  *
@@ -93,4 +95,24 @@ public class TestZKUtil {
 Assert.assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", 
"user5";
 Assert.assertTrue(aclList.contains(

hbase git commit: HBASE-18083 Make large/small file clean thread number configurable in HFileCleaner

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-2 b2b5cd6de -> 89d2adfe9


HBASE-18083 Make large/small file clean thread number configurable in 
HFileCleaner


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/89d2adfe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/89d2adfe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/89d2adfe

Branch: refs/heads/branch-2
Commit: 89d2adfe92b2e775a0a3761257846e2d4ed7d141
Parents: b2b5cd6
Author: Yu Li 
Authored: Fri Jul 7 14:07:23 2017 +0800
Committer: Yu Li 
Committed: Fri Jul 7 15:39:34 2017 +0800

--
 .../hbase/master/cleaner/HFileCleaner.java  | 154 +--
 .../hbase/master/cleaner/TestHFileCleaner.java  |  13 +-
 2 files changed, 120 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/89d2adfe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 8b3515a..6d1fbe3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -67,6 +68,16 @@ public class HFileCleaner extends 
CleanerChore impleme
   "hbase.regionserver.hfilecleaner.small.queue.size";
   public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
 
+  // Configuration key for large file delete thread number
+  public final static String LARGE_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.large.thread.count";
+  public final static int DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER = 1;
+
+  // Configuration key for small file delete thread number
+  public final static String SMALL_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.small.thread.count";
+  public final static int DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER = 1;
+
   private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
 
   StealJobQueue largeFileQueue;
@@ -74,11 +85,13 @@ public class HFileCleaner extends 
CleanerChore impleme
   private int throttlePoint;
   private int largeQueueInitSize;
   private int smallQueueInitSize;
+  private int largeFileDeleteThreadNumber;
+  private int smallFileDeleteThreadNumber;
   private List threads = new ArrayList();
   private boolean running;
 
-  private long deletedLargeFiles = 0L;
-  private long deletedSmallFiles = 0L;
+  private AtomicLong deletedLargeFiles = new AtomicLong();
+  private AtomicLong deletedSmallFiles = new AtomicLong();
 
   /**
* @param period the period of time to sleep between each run
@@ -100,6 +113,10 @@ public class HFileCleaner extends 
CleanerChore impleme
 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
 largeFileQueue = new StealJobQueue<>(largeQueueInitSize, 
smallQueueInitSize);
 smallFileQueue = largeFileQueue.getStealFromQueue();
+largeFileDeleteThreadNumber =
+conf.getInt(LARGE_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER);
+smallFileDeleteThreadNumber =
+conf.getInt(SMALL_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER);
 startHFileDeleteThreads();
   }
 
@@ -183,30 +200,34 @@ public class HFileCleaner extends 
CleanerChore impleme
 final String n = Thread.currentThread().getName();
 running = true;
 // start thread for large file deletion
-Thread large = new Thread() {
-  @Override
-  public void run() {
-consumerLoop(largeFileQueue);
-  }
-};
-large.setDaemon(true);
-large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
-large.start();
-LOG.debug("Starting hfile cleaner for large files: " + large.getName());
-threads.add(large);
+for (int i = 0; i < largeFileDeleteThreadNumber; i++) {
+  Thread large = new Thread() {
+@Override
+public void run() {
+  consumerLoop(largeFileQueue);
+}
+  };
+  large.setDaemon(true);
+  large.setName(n + "-HFileCleaner.large." + i + "-" + 
System.currentTimeMillis());
+  large.start();
+  LOG.debug("Starting hfile cleaner for large files: " + large.getName());
+  threads.add(large);
+}
 
 // start thread for small file deletion
-Thr

hbase git commit: HBASE-18083 Make large/small file clean thread number configurable in HFileCleaner

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 6786b2b63 -> 4fe738576


HBASE-18083 Make large/small file clean thread number configurable in 
HFileCleaner


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4fe73857
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4fe73857
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4fe73857

Branch: refs/heads/master
Commit: 4fe73857679ecba89a7edd3c17d9f92e4c0e2164
Parents: 6786b2b
Author: Yu Li 
Authored: Fri Jul 7 14:07:23 2017 +0800
Committer: Yu Li 
Committed: Fri Jul 7 14:07:23 2017 +0800

--
 .../hbase/master/cleaner/HFileCleaner.java  | 154 +--
 .../hbase/master/cleaner/TestHFileCleaner.java  |  13 +-
 2 files changed, 120 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4fe73857/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 8b3515a..6d1fbe3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -67,6 +68,16 @@ public class HFileCleaner extends 
CleanerChore impleme
   "hbase.regionserver.hfilecleaner.small.queue.size";
   public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
 
+  // Configuration key for large file delete thread number
+  public final static String LARGE_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.large.thread.count";
+  public final static int DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER = 1;
+
+  // Configuration key for small file delete thread number
+  public final static String SMALL_HFILE_DELETE_THREAD_NUMBER =
+  "hbase.regionserver.hfilecleaner.small.thread.count";
+  public final static int DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER = 1;
+
   private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
 
   StealJobQueue largeFileQueue;
@@ -74,11 +85,13 @@ public class HFileCleaner extends 
CleanerChore impleme
   private int throttlePoint;
   private int largeQueueInitSize;
   private int smallQueueInitSize;
+  private int largeFileDeleteThreadNumber;
+  private int smallFileDeleteThreadNumber;
   private List threads = new ArrayList();
   private boolean running;
 
-  private long deletedLargeFiles = 0L;
-  private long deletedSmallFiles = 0L;
+  private AtomicLong deletedLargeFiles = new AtomicLong();
+  private AtomicLong deletedSmallFiles = new AtomicLong();
 
   /**
* @param period the period of time to sleep between each run
@@ -100,6 +113,10 @@ public class HFileCleaner extends 
CleanerChore impleme
 conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
 largeFileQueue = new StealJobQueue<>(largeQueueInitSize, 
smallQueueInitSize);
 smallFileQueue = largeFileQueue.getStealFromQueue();
+largeFileDeleteThreadNumber =
+conf.getInt(LARGE_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER);
+smallFileDeleteThreadNumber =
+conf.getInt(SMALL_HFILE_DELETE_THREAD_NUMBER, 
DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER);
 startHFileDeleteThreads();
   }
 
@@ -183,30 +200,34 @@ public class HFileCleaner extends 
CleanerChore impleme
 final String n = Thread.currentThread().getName();
 running = true;
 // start thread for large file deletion
-Thread large = new Thread() {
-  @Override
-  public void run() {
-consumerLoop(largeFileQueue);
-  }
-};
-large.setDaemon(true);
-large.setName(n + "-HFileCleaner.large-" + System.currentTimeMillis());
-large.start();
-LOG.debug("Starting hfile cleaner for large files: " + large.getName());
-threads.add(large);
+for (int i = 0; i < largeFileDeleteThreadNumber; i++) {
+  Thread large = new Thread() {
+@Override
+public void run() {
+  consumerLoop(largeFileQueue);
+}
+  };
+  large.setDaemon(true);
+  large.setName(n + "-HFileCleaner.large." + i + "-" + 
System.currentTimeMillis());
+  large.start();
+  LOG.debug("Starting hfile cleaner for large files: " + large.getName());
+  threads.add(large);
+}
 
 // start thread for small file deletion
-Thread 

hbase git commit: HBASE-18084 Improve CleanerChore to clean from directory which consumes more disk space

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 80dd8bf51 -> 998bd5f90


HBASE-18084 Improve CleanerChore to clean from directory which consumes more 
disk space


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/998bd5f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/998bd5f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/998bd5f9

Branch: refs/heads/master
Commit: 998bd5f90e9be4a28a446045fdad38cc4b9b6b5d
Parents: 80dd8bf
Author: Yu Li 
Authored: Wed May 24 16:40:37 2017 +0800
Committer: Yu Li 
Committed: Wed May 24 16:41:04 2017 +0800

--
 .../hbase/master/cleaner/CleanerChore.java  | 73 +---
 1 file changed, 63 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/998bd5f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 825feba..1e9a4fa 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -21,6 +21,17 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -32,12 +43,6 @@ import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.ipc.RemoteException;
 
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 /**
  * Abstract Cleaner that uses a chain of delegates to clean a directory of 
files
  * @param  Cleaner delegate class that is dynamically loaded from 
configuration
@@ -151,6 +156,46 @@ public abstract class CleanerChore extends Schedu
   }
 
   /**
+   * Sort the given list in (descending) order of the space each element takes
+   * @param dirs the list to sort, element in it should be directory (not file)
+   */
+  private void sortByConsumedSpace(List dirs) {
+if (dirs == null || dirs.size() < 2) {
+  // no need to sort for empty or single directory
+  return;
+}
+Collections.sort(dirs, new Comparator() {
+  HashMap directorySpaces = new HashMap();
+
+  @Override
+  public int compare(FileStatus f1, FileStatus f2) {
+long f1ConsumedSpace = getSpace(f1);
+long f2ConsumedSpace = getSpace(f2);
+return (f1ConsumedSpace > f2ConsumedSpace) ? -1
+: (f1ConsumedSpace < f2ConsumedSpace ? 1 : 0);
+  }
+
+  private long getSpace(FileStatus f) {
+Long cached = directorySpaces.get(f);
+if (cached != null) {
+  return cached;
+}
+try {
+  long space =
+  f.isDirectory() ? 
fs.getContentSummary(f.getPath()).getSpaceConsumed() : f.getLen();
+  directorySpaces.put(f, space);
+  return space;
+} catch (IOException e) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("failed to get space consumed by path " + f.getPath(), 
e);
+  }
+  return -1;
+}
+  }
+});
+  }
+
+  /**
* Loop over the given directory entries, and check whether they can be 
deleted.
* If an entry is itself a directory it will be recursively checked and 
deleted itself iff
* all subentries are deleted (and no new subentries are added in the mean 
time)
@@ -164,16 +209,24 @@ public abstract class CleanerChore extends Schedu
 }
 boolean allEntriesDeleted = true;
 List files = Lists.newArrayListWithCapacity(entries.length);
+List dirs = new ArrayList<>();
 for (FileStatus child : entries) {
-  Path path = child.getPath();
   if (child.isDirectory()) {
+dirs.add(child);
+  } else {
+// collect all files to attempt to delete in one batch
+files.add(child);
+  }
+}
+if (dirs.size() > 0) {
+  sortByConsumedSpace(dirs);
+  LOG.debug("Prepared to delete files in directories: " + dirs);
+  for (FileStatus child : dirs) {
+Path path = child.getPath();
 

hbase git commit: HBASE-18085 Prevent parallel purge in ObjectPool

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master ebe92c8fb -> d047cc9ec


HBASE-18085 Prevent parallel purge in ObjectPool


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d047cc9e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d047cc9e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d047cc9e

Branch: refs/heads/master
Commit: d047cc9ecc7e0067c3b3aaeeb1600da8d774b211
Parents: ebe92c8
Author: Yu Li 
Authored: Wed May 24 15:42:39 2017 +0800
Committer: Yu Li 
Committed: Wed May 24 15:42:39 2017 +0800

--
 .../apache/hadoop/hbase/util/ObjectPool.java| 39 +---
 .../hadoop/hbase/util/SoftObjectPool.java   | 21 +++
 .../hadoop/hbase/util/WeakObjectPool.java   | 21 +++
 3 files changed, 44 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d047cc9e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ObjectPool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ObjectPool.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ObjectPool.java
index f736922..6a04812 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ObjectPool.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ObjectPool.java
@@ -22,6 +22,8 @@ import java.lang.ref.Reference;
 import java.lang.ref.ReferenceQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 
@@ -52,6 +54,9 @@ public abstract class ObjectPool {
   /** Does not permit null keys. */
   protected final ConcurrentMap> referenceCache;
 
+  /** For preventing parallel purge */
+  private final Lock purgeLock = new ReentrantLock();
+
   /**
* The default initial capacity,
* used when not otherwise specified in a constructor.
@@ -117,12 +122,29 @@ public abstract class ObjectPool {
   }
 
   /**
-   * Removes stale references of shared objects from the pool.
-   * References newly becoming stale may still remain.
-   * The implementation of this method is expected to be lightweight
-   * when there is no stale reference.
+   * Removes stale references of shared objects from the pool. References 
newly becoming stale may
+   * still remain.
+   * 
+   * The implementation of this method is expected to be lightweight when 
there is no stale
+   * reference with the Oracle (Sun) implementation of {@code ReferenceQueue}, 
because
+   * {@code ReferenceQueue.poll} just checks a volatile instance variable in 
{@code ReferenceQueue}.
*/
-  public abstract void purge();
+  public void purge() {
+if (purgeLock.tryLock()) {// no parallel purge
+  try {
+while (true) {
+  @SuppressWarnings("unchecked")
+  Reference ref = (Reference) staleRefQueue.poll();
+  if (ref == null) {
+break;
+  }
+  referenceCache.remove(getReferenceKey(ref), ref);
+}
+  } finally {
+purgeLock.unlock();
+  }
+}
+  }
 
   /**
* Create a reference associated with the given object
@@ -133,6 +155,13 @@ public abstract class ObjectPool {
   public abstract Reference createReference(K key, V obj);
 
   /**
+   * Get key of the given reference
+   * @param ref The reference
+   * @return key of the reference
+   */
+  public abstract K getReferenceKey(Reference ref);
+
+  /**
* Returns a shared object associated with the given {@code key},
* which is identified by the {@code equals} method.
* @throws NullPointerException if {@code key} is null

http://git-wip-us.apache.org/repos/asf/hbase/blob/d047cc9e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/SoftObjectPool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/SoftObjectPool.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/SoftObjectPool.java
index 7f27f98..19c65b2 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/SoftObjectPool.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/SoftObjectPool.java
@@ -49,22 +49,6 @@ public class SoftObjectPool extends ObjectPool {
   }
 
   @Override
-  public void purge() {
-// This method is lightweight while there is no stale reference
-// with the Oracle (Sun) implementation of {@code ReferenceQueue},
-// because {@code ReferenceQueue.poll} just checks a volatile instance
-// variable in {@code ReferenceQueue}.
-while (true) {
-  @SuppressWarnings("unchecked")
-  SoftObjectReference ref = (SoftO

hbase git commit: HBASE-17471 Region Seqid will be out of order in WAL if using mvccPreAssign

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 408645c4e -> 9a97e28bd


HBASE-17471 Region Seqid will be out of order in WAL if using mvccPreAssign

Signed-off-by: Yu Li 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a97e28b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a97e28b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a97e28b

Branch: refs/heads/branch-1
Commit: 9a97e28bdf202ea2a8cadffbb8f8de4c1411447e
Parents: 408645c
Author: Allan Yang 
Authored: Sat May 6 23:28:14 2017 +0800
Committer: Yu Li 
Committed: Sat May 6 23:29:49 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  | 66 +
 .../MultiVersionConcurrencyControl.java | 32 +---
 .../hadoop/hbase/regionserver/wal/FSHLog.java   | 45 +--
 .../hbase/regionserver/wal/FSWALEntry.java  | 27 ++-
 .../org/apache/hadoop/hbase/wal/WALKey.java | 78 +---
 .../hbase/coprocessor/TestWALObserver.java  |  6 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |  7 +-
 .../hbase/regionserver/TestWALLockup.java   | 12 +--
 .../hbase/regionserver/wal/TestFSHLog.java  |  1 +
 .../wal/TestWALActionsListener.java |  5 +-
 .../hbase/regionserver/wal/TestWALReplay.java   |  2 +-
 .../apache/hadoop/hbase/wal/TestSecureWAL.java  |  4 +-
 .../apache/hadoop/hbase/wal/TestWALFactory.java |  4 +-
 13 files changed, 92 insertions(+), 197 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a97e28b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 1fbb9b2..8e1306f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -664,10 +664,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   private final Durability durability;
   private final boolean regionStatsEnabled;
 
-  // flag and lock for MVCC preassign
-  private final boolean mvccPreAssign;
-  private final ReentrantLock preAssignMvccLock;
-
   // whether to unassign region if we hit FNFE
   private final RegionUnassigner regionUnassigner;
   /**
@@ -820,13 +816,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   conf.getBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE,
   HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);
 
-// get mvcc pre-assign flag and lock
-this.mvccPreAssign = conf.getBoolean(HREGION_MVCC_PRE_ASSIGN, 
DEFAULT_HREGION_MVCC_PRE_ASSIGN);
-if (this.mvccPreAssign) {
-  this.preAssignMvccLock = new ReentrantLock();
-} else {
-  this.preAssignMvccLock = null;
-}
 boolean unassignForFNFE =
 conf.getBoolean(HREGION_UNASSIGN_FOR_FNFE, 
DEFAULT_HREGION_UNASSIGN_FOR_FNFE);
 if (unassignForFNFE) {
@@ -2674,9 +2663,9 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 // a timeout. May happen in tests after we tightened the semantic via 
HBASE-14317.
 // Also, the getSequenceId blocks on a latch. There is no global list of 
outstanding latches
 // so if an abort or stop, there is no way to call them in.
-WALKey key = this.appendEmptyEdit(wal, null);
+WALKey key = this.appendEmptyEdit(wal);
 mvcc.complete(key.getWriteEntry());
-return key.getSequenceId(this.maxWaitForSeqId);
+return key.getSequenceId();
   }
 
   
//
@@ -3418,29 +3407,16 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   this.wal.append(this.htableDescriptor, this.getRegionInfo(), 
walKey, walEdit, true);
 }
   } else {
-try {
-  if (mvccPreAssign) {
-preAssignMvccLock.lock();
-writeEntry = mvcc.begin();
-  }
-  if (walEdit.size() > 0) {
-// we use HLogKey here instead of WALKey directly to support 
legacy coprocessors.
-walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
-this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, 
now,
-mutation.getClusterIds(), currentNonceGroup, currentNonce, 
mvcc);
-if (mvccPreAssign) {
-  walKey.setPreAssignedWriteEntry(writeEntry);
-}
-txid =
-this.wal.append(this.htableDescriptor, this.getRegionInfo(), 
walKey, walEdit, true);
-  } else {
-

hbase git commit: HBASE-17886 Fix compatibility of ServerSideScanMetrics

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 cd583383b -> 3fb5f3b08


HBASE-17886 Fix compatibility of ServerSideScanMetrics


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3fb5f3b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3fb5f3b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3fb5f3b0

Branch: refs/heads/branch-1.2
Commit: 3fb5f3b083e5f506604df2a7f73dd8c9cb4fcd5d
Parents: cd58338
Author: Yu Li 
Authored: Thu Apr 6 17:29:22 2017 +0800
Committer: Yu Li 
Committed: Sat Apr 8 00:20:06 2017 +0800

--
 .../hadoop/hbase/client/metrics/ServerSideScanMetrics.java  | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3fb5f3b0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
index 4b3e0ce..d659766 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
@@ -51,6 +51,11 @@ public class ServerSideScanMetrics {
   public static final String COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME = 
"ROWS_SCANNED";
   public static final String COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME = 
"ROWS_FILTERED";
 
+  /** @deprecated Use {@link #COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME} instead */
+  public static final String COUNT_OF_ROWS_SCANNED_KEY = 
COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
+  /** @deprecated Use {@link #COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME} instead 
*/
+  public static final String COUNT_OF_ROWS_FILTERED_KEY = 
COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME;
+
   /**
* number of rows filtered during scan RPC
*/

hbase git commit: HBASE-17886 Fix compatibility of ServerSideScanMetrics

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 4057a6c89 -> 96890d64d


HBASE-17886 Fix compatibility of ServerSideScanMetrics


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96890d64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96890d64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96890d64

Branch: refs/heads/branch-1
Commit: 96890d64d41a3263a54becabd0c0079d67a1a8b7
Parents: 4057a6c
Author: Yu Li 
Authored: Thu Apr 6 17:29:22 2017 +0800
Committer: Yu Li 
Committed: Thu Apr 6 18:19:34 2017 +0800

--
 .../hadoop/hbase/client/metrics/ServerSideScanMetrics.java  | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/96890d64/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
index 6e4e9b8..7cf7fb4 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
@@ -51,6 +51,11 @@ public class ServerSideScanMetrics {
   public static final String COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME = 
"ROWS_SCANNED";
   public static final String COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME = 
"ROWS_FILTERED";
 
+  /** @deprecated Use {@link #COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME} instead */
+  public static final String COUNT_OF_ROWS_SCANNED_KEY = 
COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
+  /** @deprecated Use {@link #COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME} instead 
*/
+  public static final String COUNT_OF_ROWS_FILTERED_KEY = 
COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME;
+
   /**
* number of rows filtered during scan RPC
*/

hbase git commit: HBASE-17886 Fix compatibility of ServerSideScanMetrics

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 2a425fc42 -> 627f0796a


HBASE-17886 Fix compatibility of ServerSideScanMetrics


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/627f0796
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/627f0796
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/627f0796

Branch: refs/heads/branch-1.3
Commit: 627f0796af07f176c6df1501529c39b7bccd8044
Parents: 2a425fc
Author: Yu Li 
Authored: Thu Apr 6 17:29:22 2017 +0800
Committer: Yu Li 
Committed: Thu Apr 6 17:31:03 2017 +0800

--
 .../hadoop/hbase/client/metrics/ServerSideScanMetrics.java  | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/627f0796/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
index 4b3e0ce..d659766 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
@@ -51,6 +51,11 @@ public class ServerSideScanMetrics {
   public static final String COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME = 
"ROWS_SCANNED";
   public static final String COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME = 
"ROWS_FILTERED";
 
+  /** @deprecated Use {@link #COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME} instead */
+  public static final String COUNT_OF_ROWS_SCANNED_KEY = 
COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
+  /** @deprecated Use {@link #COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME} instead 
*/
+  public static final String COUNT_OF_ROWS_FILTERED_KEY = 
COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME;
+
   /**
* number of rows filtered during scan RPC
*/

hbase git commit: HBASE-17886 Fix compatibility of ServerSideScanMetrics

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 17737b271 -> d7e3116a1


HBASE-17886 Fix compatibility of ServerSideScanMetrics


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d7e3116a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d7e3116a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d7e3116a

Branch: refs/heads/master
Commit: d7e3116a1744057359ca48d94aa50d7fdf0db974
Parents: 17737b2
Author: Yu Li 
Authored: Thu Apr 6 17:29:22 2017 +0800
Committer: Yu Li 
Committed: Thu Apr 6 17:29:22 2017 +0800

--
 .../hadoop/hbase/client/metrics/ServerSideScanMetrics.java  | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d7e3116a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
index 8a96aeb..03764ed 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
@@ -49,6 +49,11 @@ public class ServerSideScanMetrics {
   public static final String COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME = 
"ROWS_SCANNED";
   public static final String COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME = 
"ROWS_FILTERED";
 
+  /** @deprecated Use {@link #COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME} instead */
+  public static final String COUNT_OF_ROWS_SCANNED_KEY = 
COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
+  /** @deprecated Use {@link #COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME} instead 
*/
+  public static final String COUNT_OF_ROWS_FILTERED_KEY = 
COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME;
+
   /**
* number of rows filtered during scan RPC
*/

hbase git commit: HBASE-17854 Use StealJobQueue in HFileCleaner after HBASE-17215

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master a66d49189 -> cbcbcf4dc


HBASE-17854 Use StealJobQueue in HFileCleaner after HBASE-17215


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbcbcf4d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbcbcf4d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbcbcf4d

Branch: refs/heads/master
Commit: cbcbcf4dcd3401327cc36173f3ca8e5362da1e0c
Parents: a66d491
Author: Yu Li 
Authored: Wed Apr 5 17:53:21 2017 +0800
Committer: Yu Li 
Committed: Wed Apr 5 17:53:21 2017 +0800

--
 .../hbase/master/cleaner/HFileCleaner.java  | 98 +---
 .../apache/hadoop/hbase/util/StealJobQueue.java | 22 +
 .../hbase/master/cleaner/TestHFileCleaner.java  | 28 +++---
 .../hadoop/hbase/util/TestStealJobQueue.java|  2 +-
 4 files changed, 102 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cbcbcf4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 3a68252..8b3515a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -35,6 +34,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.StealJobQueue;
 
 import com.google.common.annotations.VisibleForTesting;
 /**
@@ -57,23 +57,23 @@ public class HFileCleaner extends 
CleanerChore impleme
   "hbase.regionserver.thread.hfilecleaner.throttle";
   public final static int DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD = 64 * 1024 
* 1024;// 64M
 
-  // Configuration key for large queue size
-  public final static String LARGE_HFILE_DELETE_QUEUE_SIZE =
+  // Configuration key for large queue initial size
+  public final static String LARGE_HFILE_QUEUE_INIT_SIZE =
   "hbase.regionserver.hfilecleaner.large.queue.size";
-  public final static int DEFAULT_LARGE_HFILE_DELETE_QUEUE_SIZE = 1048576;
+  public final static int DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE = 10240;
 
-  // Configuration key for small queue size
-  public final static String SMALL_HFILE_DELETE_QUEUE_SIZE =
+  // Configuration key for small queue initial size
+  public final static String SMALL_HFILE_QUEUE_INIT_SIZE =
   "hbase.regionserver.hfilecleaner.small.queue.size";
-  public final static int DEFAULT_SMALL_HFILE_DELETE_QUEUE_SIZE = 1048576;
+  public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
 
   private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
 
-  BlockingQueue largeFileQueue;
+  StealJobQueue largeFileQueue;
   BlockingQueue smallFileQueue;
   private int throttlePoint;
-  private int largeQueueSize;
-  private int smallQueueSize;
+  private int largeQueueInitSize;
+  private int smallQueueInitSize;
   private List threads = new ArrayList();
   private boolean running;
 
@@ -94,12 +94,12 @@ public class HFileCleaner extends 
CleanerChore impleme
   directory, MASTER_HFILE_CLEANER_PLUGINS, params);
 throttlePoint =
 conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, 
DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
-largeQueueSize =
-conf.getInt(LARGE_HFILE_DELETE_QUEUE_SIZE, 
DEFAULT_LARGE_HFILE_DELETE_QUEUE_SIZE);
-smallQueueSize =
-conf.getInt(SMALL_HFILE_DELETE_QUEUE_SIZE, 
DEFAULT_SMALL_HFILE_DELETE_QUEUE_SIZE);
-largeFileQueue = new 
LinkedBlockingQueue(largeQueueSize);
-smallFileQueue = new 
LinkedBlockingQueue(smallQueueSize);
+largeQueueInitSize =
+conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
+smallQueueInitSize =
+conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, 
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
+largeFileQueue = new StealJobQueue<>(largeQueueInitSize, 
smallQueueInitSize);
+smallFileQueue = largeFileQueue.getStealFromQueue();
 startHFileDeleteThreads();
   }
 
@@ -152,6 +152,7 @@ public class HFileCleaner extends 
CleanerChore impleme
   private boolean dispatch(HFileDeleteTask task) {
 if (task.fileLength >= this.throttlePoint) {
   if (!this.largeFileQueue.offe

hbase git commit: HBASE-17215 Separate small/large file delete threads in HFileCleaner to accelerate archived hfile cleanup speed

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 5f98ad205 -> 9facfa550


HBASE-17215 Separate small/large file delete threads in HFileCleaner to 
accelerate archived hfile cleanup speed


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9facfa55
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9facfa55
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9facfa55

Branch: refs/heads/master
Commit: 9facfa550f1e7386be3a04d84f7e8013f5002965
Parents: 5f98ad2
Author: Yu Li 
Authored: Sat Apr 1 10:59:11 2017 +0800
Committer: Yu Li 
Committed: Sat Apr 1 10:59:11 2017 +0800

--
 .../org/apache/hadoop/hbase/master/HMaster.java |   1 +
 .../hbase/master/cleaner/CleanerChore.java  |  14 +-
 .../hbase/master/cleaner/HFileCleaner.java  | 313 ++-
 .../hbase/regionserver/RSRpcServices.java   |   3 +-
 .../hbase/master/cleaner/TestHFileCleaner.java  | 152 +
 5 files changed, 478 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9facfa55/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a1cbe53..bb9f282 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -885,6 +885,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 status.markComplete("Initialization successful");
 LOG.info("Master has completed initialization");
 configurationManager.registerObserver(this.balancer);
+configurationManager.registerObserver(this.hfileCleaner);
 
 // Set master as 'initialized'.
 setInitialized(true);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9facfa55/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index dddad36..825feba 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -46,7 +46,7 @@ public abstract class CleanerChore extends Schedu
 
   private static final Log LOG = 
LogFactory.getLog(CleanerChore.class.getName());
 
-  private final FileSystem fs;
+  protected final FileSystem fs;
   private final Path oldFileDir;
   private final Configuration conf;
   protected List cleanersChain;
@@ -269,6 +269,15 @@ public abstract class CleanerChore extends Schedu
 }
 
 Iterable filesToDelete = Iterables.concat(invalidFiles, 
deletableValidFiles);
+return deleteFiles(filesToDelete) == files.size();
+  }
+
+  /**
+   * Delete the given files
+   * @param filesToDelete files to delete
+   * @return number of deleted files
+   */
+  protected int deleteFiles(Iterable filesToDelete) {
 int deletedFileCount = 0;
 for (FileStatus file : filesToDelete) {
   Path filePath = file.getPath();
@@ -289,8 +298,7 @@ public abstract class CleanerChore extends Schedu
 LOG.warn("Error while deleting: " + filePath, e);
   }
 }
-
-return deletedFileCount == files.size();
+return deletedFileCount;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/9facfa55/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 89c316b..3a68252 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -17,22 +17,33 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus

hbase git commit: HBASE-17813 backport HBASE-16983 to branch-1.3

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 a60792425 -> ab335bf9d


HBASE-17813 backport HBASE-16983 to branch-1.3

This fixes the "Unable to create region directory" issue in 
TestMultiTableSnapshotInputFormat


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ab335bf9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ab335bf9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ab335bf9

Branch: refs/heads/branch-1.3
Commit: ab335bf9d3d82100a875c796eea8e9532b9d2d7b
Parents: a607924
Author: Yu Li 
Authored: Tue Mar 21 21:18:32 2017 +0800
Committer: Yu Li 
Committed: Tue Mar 21 21:18:32 2017 +0800

--
 .../hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java | 3 +--
 .../org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java   | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ab335bf9/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java
index 93bb820..1d55957 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java
@@ -63,8 +63,7 @@ public class TestMultiTableSnapshotInputFormat extends 
MultiTableInputFormatTest
 
   @Before
   public void setUp() throws Exception {
-this.restoreDir = new Path("/tmp");
-
+this.restoreDir = TEST_UTIL.getRandomDir();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/ab335bf9/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
index 5aa96c1..c5200fb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java
@@ -66,7 +66,7 @@ public class TestStoreFileInfo {
   @Test
   public void testEqualsWithLink() throws IOException {
 Path origin = new Path("/origin");
-Path tmp = new Path("/tmp");
+Path tmp = TEST_UTIL.getDataTestDir();
 Path archive = new Path("/archive");
 HFileLink link1 = new HFileLink(new Path(origin, "f1"), new Path(tmp, 
"f1"),
   new Path(archive, "f1"));

hbase git commit: HBASE-17070 backport HBASE-17020 (keylen in midkey() dont computed correctly) to 1.3.1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 693b51d81 -> a60792425


HBASE-17070 backport HBASE-17020 (keylen in midkey() dont computed correctly) 
to 1.3.1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a6079242
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a6079242
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a6079242

Branch: refs/heads/branch-1.3
Commit: a60792425a50de48d6af88ff2737b5e32413de8a
Parents: 693b51d
Author: Yu Li 
Authored: Tue Mar 21 21:15:16 2017 +0800
Committer: Yu Li 
Committed: Tue Mar 21 21:15:16 2017 +0800

--
 .../hadoop/hbase/io/hfile/HFileBlockIndex.java  |  2 +-
 .../hbase/io/hfile/TestHFileBlockIndex.java | 68 
 .../hbase/io/hfile/TestHFileWriterV2.java   | 29 +
 3 files changed, 98 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a6079242/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index e24b4f1..0d0a42c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -355,7 +355,7 @@ public class HFileBlockIndex {
 int numDataBlocks = b.getInt();
 int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1));
 int keyLen = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 2)) -
-keyRelOffset;
+keyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
 int keyOffset = Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset
 + SECONDARY_INDEX_ENTRY_OVERHEAD;
 targetMidKey = ByteBufferUtils.toBytes(b, keyOffset, keyLen);

http://git-wip-us.apache.org/repos/asf/hbase/blob/a6079242/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index b3e0ade..6372713 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
@@ -502,6 +503,73 @@ public class TestHFileBlockIndex {
   }
 
   /**
+  * to check if looks good when midKey on a leaf index block boundary
+  * @throws IOException
+  */
+  @Test
+ public void testMidKeyOnLeafIndexBlockBoundary() throws IOException {
+   Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
+   "hfile_for_midkey");
+   int maxChunkSize = 512;
+   conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize);
+   // should open hfile.block.index.cacheonwrite
+   conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true);
+
+   CacheConfig cacheConf = new CacheConfig(conf);
+   BlockCache blockCache = cacheConf.getBlockCache();
+   // Evict all blocks that were cached-on-write by the previous invocation.
+   blockCache.evictBlocksByHfileName(hfilePath.getName());
+   // Write the HFile
+   {
+ HFileContext meta = new HFileContextBuilder()
+ .withBlockSize(SMALL_BLOCK_SIZE)
+ .withCompression(Algorithm.NONE)
+ .withDataBlockEncoding(DataBlockEncoding.NONE)
+ .build();
+ HFile.Writer writer =
+   HFile.getWriterFactory(conf, cacheConf)
+   .withPath(fs, hfilePath)
+   .withFileContext(meta)
+   .create();
+ Random rand = new Random(19231737);
+ byte[] family = Bytes.toBytes("f");
+ byte[] qualifier = Bytes.toBytes("q");
+ int kvNumberToBeWritten = 16;
+ // the new generated hfile will contain 2 leaf-index blocks and 16 data 
blocks,
+ // midkey is just on the boundary of the first leaf-index block
+ for (int i = 0; i < kvNumberToBeWritten; ++i) {
+   byte[] row = TestHFileWriterV2.randomOrderedFi

hbase git commit: HBASE-17060 backport HBASE-16570 (Compute region locality in parallel at startup) to 1.3.1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 446a21fed -> 693b51d81


HBASE-17060 backport HBASE-16570 (Compute region locality in parallel at 
startup) to 1.3.1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/693b51d8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/693b51d8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/693b51d8

Branch: refs/heads/branch-1.3
Commit: 693b51d81af0c446b305af69fe130faee07581a6
Parents: 446a21f
Author: Yu Li 
Authored: Tue Mar 21 15:14:31 2017 +0800
Committer: Yu Li 
Committed: Tue Mar 21 15:14:31 2017 +0800

--
 .../hbase/master/balancer/BaseLoadBalancer.java | 11 +++--
 .../master/balancer/RegionLocationFinder.java   | 47 ++--
 .../balancer/TestRegionLocationFinder.java  | 21 +
 3 files changed, 71 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/693b51d8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index c2529a8..2df4fbe 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -1231,7 +1231,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   return assignments;
 }
 
-Cluster cluster = createCluster(servers, regions);
+Cluster cluster = createCluster(servers, regions, false);
 List unassignedRegions = new ArrayList();
 
 roundRobinAssignment(cluster, regions, unassignedRegions,
@@ -1278,7 +1278,10 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   }
 
   protected Cluster createCluster(List servers,
-  Collection regions) {
+  Collection regions, boolean forceRefresh) {
+if (forceRefresh) {
+  regionFinder.refreshAndWait(regions);
+}
 // Get the snapshot of the current assignments for the regions in 
question, and then create
 // a cluster out of it. Note that we might have replicas already assigned 
to some servers
 // earlier. So we want to get the snapshot to see those assignments, but 
this will only contain
@@ -1352,7 +1355,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 }
 
 List regions = Lists.newArrayList(regionInfo);
-Cluster cluster = createCluster(servers, regions);
+Cluster cluster = createCluster(servers, regions, false);
 return randomAssignment(cluster, regionInfo, servers);
   }
 
@@ -1427,7 +1430,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 int numRandomAssignments = 0;
 int numRetainedAssigments = 0;
 
-Cluster cluster = createCluster(servers, regions.keySet());
+Cluster cluster = createCluster(servers, regions.keySet(), true);
 
 for (Map.Entry entry : regions.entrySet()) {
   HRegionInfo region = entry.getKey();

http://git-wip-us.apache.org/repos/asf/hbase/blob/693b51d8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index a6724ee..6c5cb19 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -21,6 +21,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -63,11 +64,13 @@ import java.util.concurrent.TimeUnit;
 class RegionLocationFinder {
   private static final Log LOG = LogFactory.getLog(RegionLocationFinder.class);
   private static final long CACHE_TIME = 240 * 60 * 1000;
+  private static final HDFSBlocksDistribution EMPTY_BLOCK_DISTRIBUTION = new 
HDFSBlocksDistribution();
   private Configuration conf;
   private volatile ClusterStatus status;
   private MasterServices services;
   private final ListeningExecutorService executor;
-  private long lastFullRefresh = 0;
+  // Do not sch

hbase git commit: HBASE-17059 backport HBASE-17039 (SimpleLoadBalancer schedules large amount of invalid region moves) to 1.3.1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 98b5d2cd4 -> 446a21fed


HBASE-17059 backport HBASE-17039 (SimpleLoadBalancer schedules large amount of 
invalid region moves) to 1.3.1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/446a21fe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/446a21fe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/446a21fe

Branch: refs/heads/branch-1.3
Commit: 446a21fedd1282c15939eb4c46d13c859beedd7a
Parents: 98b5d2c
Author: Yu Li 
Authored: Tue Mar 21 14:25:58 2017 +0800
Committer: Yu Li 
Committed: Tue Mar 21 14:27:05 2017 +0800

--
 .../hadoop/hbase/master/balancer/SimpleLoadBalancer.java   | 6 +-
 1 file changed, 1 insertion(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/446a21fe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
index 4325585..a354e40 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
@@ -273,14 +273,10 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
 serversByLoad.entrySet()) {
   if (maxToTake == 0) break; // no more to take
   int load = server.getKey().getLoad();
-  if (load >= min && load > 0) {
+  if (load >= min) {
 continue; // look for other servers which haven't reached min
   }
   int regionsToPut = min - load;
-  if (regionsToPut == 0)
-  {
-regionsToPut = 1;
-  }
   maxToTake -= regionsToPut;
   underloadedServers.put(server.getKey().getServerName(), regionsToPut);
 }

hbase git commit: HBASE-17175 backport HBASE-17127 (Locate region should fail fast if underlying Connection already closed) to 1.3.1

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 226b6fa44 -> 98b5d2cd4


HBASE-17175 backport HBASE-17127 (Locate region should fail fast if underlying 
Connection already closed) to 1.3.1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/98b5d2cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/98b5d2cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/98b5d2cd

Branch: refs/heads/branch-1.3
Commit: 98b5d2cd4b0badef5f576cb35c8c8f1df1c88899
Parents: 226b6fa
Author: Yu Li 
Authored: Tue Mar 21 06:05:29 2017 +0800
Committer: Yu Li 
Committed: Tue Mar 21 06:05:29 2017 +0800

--
 .../hadoop/hbase/client/ConnectionManager.java  |  2 +-
 .../hbase/client/TestClientNoCluster.java   | 27 
 2 files changed, 28 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/98b5d2cd/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 4e9d208..cba441e 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -1171,7 +1171,7 @@ class ConnectionManager {
 public RegionLocations locateRegion(final TableName tableName,
   final byte [] row, boolean useCache, boolean retry, int replicaId)
 throws IOException {
-  if (this.closed) throw new IOException(toString() + " closed");
+  if (this.closed) throw new DoNotRetryIOException(toString() + " closed");
   if (tableName== null || tableName.getName().length == 0) {
 throw new IllegalArgumentException(
 "table name cannot be null or zero length");

http://git-wip-us.apache.org/repos/asf/hbase/blob/98b5d2cd/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
index 2d50c1b..f6968bc 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
@@ -39,6 +39,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -286,6 +287,32 @@ public class TestClientNoCluster extends Configured 
implements Tool {
 }
   }
 
+  @Test
+  public void testConnectionClosedOnRegionLocate() throws IOException {
+Configuration testConf = new Configuration(this.conf);
+testConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+// Go against meta else we will try to find first region for the table on 
construction which
+// means we'll have to do a bunch more mocking. Tests that go against meta 
only should be
+// good for a bit of testing.
+Connection connection = ConnectionFactory.createConnection(testConf);
+Table table = connection.getTable(TableName.META_TABLE_NAME);
+connection.close();
+try {
+  Get get = new Get(Bytes.toBytes("dummyRow"));
+  table.get(get);
+  fail("Should have thrown DoNotRetryException but no exception thrown");
+} catch (Exception e) {
+  if (!(e instanceof DoNotRetryIOException)) {
+String errMsg =
+"Should have thrown DoNotRetryException but actually " + 
e.getClass().getSimpleName();
+LOG.error(errMsg, e);
+fail(errMsg);
+  }
+} finally {
+  table.close();
+}
+  }
+
   /**
* Override to shutdown going to zookeeper for cluster id and meta location.
*/

hbase git commit: HBASE-17782 Extend IdReadWriteLock to support using both weak and soft reference

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 14fb57cab -> aace02a23


HBASE-17782 Extend IdReadWriteLock to support using both weak and soft reference


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aace02a2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aace02a2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aace02a2

Branch: refs/heads/master
Commit: aace02a230a61cc7e91eb240598435c36c9af403
Parents: 14fb57c
Author: Yu Li 
Authored: Wed Mar 15 11:07:42 2017 +0800
Committer: Yu Li 
Committed: Wed Mar 15 11:07:42 2017 +0800

--
 .../hbase/io/hfile/bucket/BucketCache.java  |  5 +-
 .../hadoop/hbase/util/IdReadWriteLock.java  | 58 
 .../hbase/wal/RegionGroupingProvider.java   | 13 +++--
 .../hadoop/hbase/util/TestIdReadWriteLock.java  | 31 +--
 4 files changed, 86 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/aace02a2/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index cb23ca9..3e9c376 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.hbase.nio.ByteBuff;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.HasThread;
 import org.apache.hadoop.hbase.util.IdReadWriteLock;
+import org.apache.hadoop.hbase.util.IdReadWriteLock.ReferenceType;
 import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -185,9 +186,11 @@ public class BucketCache implements BlockCache, HeapSize {
   /**
* A ReentrantReadWriteLock to lock on a particular block identified by 
offset.
* The purpose of this is to avoid freeing the block which is being read.
+   * 
+   * Key set of offsets in BucketCache is limited so soft reference is the 
best choice here.
*/
   @VisibleForTesting
-  final IdReadWriteLock offsetLock = new IdReadWriteLock();
+  final IdReadWriteLock offsetLock = new IdReadWriteLock(ReferenceType.SOFT);
 
   private final NavigableSet blocksByHFile =
   new ConcurrentSkipListSet<>(new Comparator() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/aace02a2/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java
index deb2265..2a83029 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdReadWriteLock.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.lang.ref.Reference;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -44,16 +45,48 @@ import com.google.common.annotations.VisibleForTesting;
 public class IdReadWriteLock {
   // The number of lock we want to easily support. It's not a maximum.
   private static final int NB_CONCURRENT_LOCKS = 1000;
-  // The pool to get entry from, entries are mapped by soft reference and will 
be
-  // automatically garbage-collected when JVM memory pressure is high
-  private final ObjectPool lockPool =
-  new SoftObjectPool<>(
-  new ObjectPool.ObjectFactory() {
-@Override
-public ReentrantReadWriteLock createObject(Long id) {
-  return new ReentrantReadWriteLock();
-}
-  }, NB_CONCURRENT_LOCKS);
+  /**
+   * The pool to get entry from, entries are mapped by {@link Reference} and 
will be automatically
+   * garbage-collected by JVM
+   */
+  private final ObjectPool lockPool;
+  private final ReferenceType refType;
+
+  public IdReadWriteLock() {
+this(ReferenceType.WEAK);
+  }
+
+  /**
+   * Constructor of IdReadWriteLock
+   * @param referenceType type of the reference used in lock pool, {@link 
ReferenceType#WEAK} by
+   *  default. Use {@link ReferenceType#SOFT} if the key set is 
limited and the locks will
+   *  be reused with a high frequency
+   */
+  public IdReadWriteLock(ReferenceType referenceType) {
+this.refType = referenceType;
+switch (referenceType) {
+case SOFT:
+  lockPool = new SoftObjectPool<>(new ObjectPool.Ob