hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 2ee7d2a48 -> 1972996ff


HBASE-15849 Simplify the way we handle runtime of commands.
Functions format_simple_command and format_and_return_simple_command are used 
to print runtimes right now. They are called from within every single command 
and use Ruby's 'yield' magic.  Instead, we can simplify it using 'command_safe' 
function. Since command_safe wraps all commands, we can simply time before and 
after we call individual command.
If a command only wants to time a part of its logic, it can set instance 
variables start_time and end_time accordingly which is far more simpler to 
understand and work with than 'yield'.

Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1972996f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1972996f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1972996f

Branch: refs/heads/branch-1
Commit: 1972996ffcbcbd13327c277321a687e7559991ca
Parents: 2ee7d2a
Author: Apekshit Sharma 
Authored: Fri Jun 3 14:13:45 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 16:59:34 2016 -0700

--
 hbase-shell/src/main/ruby/shell.rb  |  7 ++---
 hbase-shell/src/main/ruby/shell/commands.rb | 27 +++-
 .../main/ruby/shell/commands/abort_procedure.rb |  6 +
 .../src/main/ruby/shell/commands/add_labels.rb  |  4 +--
 .../src/main/ruby/shell/commands/add_peer.rb|  4 +--
 .../src/main/ruby/shell/commands/alter.rb   |  4 +--
 .../src/main/ruby/shell/commands/alter_async.rb |  4 +--
 .../main/ruby/shell/commands/alter_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/append.rb  | 10 
 .../ruby/shell/commands/append_peer_tableCFs.rb |  4 +--
 .../src/main/ruby/shell/commands/assign.rb  |  4 +--
 .../main/ruby/shell/commands/balance_switch.rb  |  6 +
 .../src/main/ruby/shell/commands/balancer.rb| 14 +-
 .../ruby/shell/commands/balancer_enabled.rb |  6 +
 .../shell/commands/catalogjanitor_enabled.rb|  6 +
 .../ruby/shell/commands/catalogjanitor_run.rb   |  4 +--
 .../shell/commands/catalogjanitor_switch.rb |  6 +
 .../src/main/ruby/shell/commands/clear_auths.rb |  4 +--
 .../main/ruby/shell/commands/clone_snapshot.rb  |  4 +--
 .../main/ruby/shell/commands/close_region.rb|  4 +--
 .../src/main/ruby/shell/commands/compact.rb |  4 +--
 .../src/main/ruby/shell/commands/compact_rs.rb  |  4 +--
 .../src/main/ruby/shell/commands/count.rb   |  4 +--
 .../src/main/ruby/shell/commands/create.rb  |  7 +++--
 .../ruby/shell/commands/create_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/delete.rb  | 13 +-
 .../ruby/shell/commands/delete_all_snapshot.rb  |  6 ++---
 .../main/ruby/shell/commands/delete_snapshot.rb |  4 +--
 .../shell/commands/delete_table_snapshots.rb| 19 +++---
 .../src/main/ruby/shell/commands/deleteall.rb   |  5 ++--
 .../src/main/ruby/shell/commands/describe.rb|  4 +--
 .../ruby/shell/commands/describe_namespace.rb   |  3 ---
 .../src/main/ruby/shell/commands/disable.rb |  4 +--
 .../main/ruby/shell/commands/disable_peer.rb|  4 +--
 .../shell/commands/disable_table_replication.rb |  6 ++---
 .../src/main/ruby/shell/commands/drop.rb|  4 +--
 .../main/ruby/shell/commands/drop_namespace.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable_peer.rb |  4 +--
 .../shell/commands/enable_table_replication.rb  |  6 ++---
 .../src/main/ruby/shell/commands/exists.rb  |  4 +--
 .../src/main/ruby/shell/commands/flush.rb   |  4 +--
 hbase-shell/src/main/ruby/shell/commands/get.rb |  4 +--
 .../src/main/ruby/shell/commands/get_auths.rb   |  8 +++---
 .../main/ruby/shell/commands/get_peer_config.rb |  7 +++--
 .../src/main/ruby/shell/commands/get_table.rb   |  4 +--
 .../src/main/ruby/shell/commands/grant.rb   |  9 +++
 .../src/main/ruby/shell/commands/incr.rb| 12 -
 .../src/main/ruby/shell/commands/is_disabled.rb |  8 ++
 .../src/main/ruby/shell/commands/is_enabled.rb  |  6 +
 .../src/main/ruby/shell/commands/list.rb|  3 +--
 .../src/main/ruby/shell/commands/list_labels.rb |  8 +++---
 .../main/ruby/shell/commands/list_namespace.rb  |  3 +--
 .../shell/commands/list_namespace_tables.rb |  3 +--
 .../ruby/shell/commands/list_peer_configs.rb| 18 ++---
 .../src/main/ruby/shell/commands/list_peers.rb  |  3 +--
 .../main/ruby/shell/commands/list_procedures.rb |  3 +--
 .../src/main/ruby/shell/commands/list_quotas.rb |  3 +--
 .../shell/commands/list_replicated_tables.rb|  4 +--
 .../main/ruby/shell/commands/list_snapshots.rb  |  3 +--
 .../ruby/shell/commands/list_table_snapshots.rb |  3 +--

hbase git commit: HBASE-15845 Changes: - Remove omnipresence of formatter object since it is kind of a use-and-throw class. Commands should create an instance, use it to format the output and discard

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 fa182c2ea -> 2ee7d2a48


HBASE-15845 Changes:
- Remove omnipresence of formatter object since it is kind of a use-and-throw 
class. Commands should create
  an instance, use it to format the output and discard it.
- Some refactoring
Tested: Ran TestShell

Change-Id: I12176ee12695df952ee4b0f8397e5d3be8f16a87


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2ee7d2a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2ee7d2a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2ee7d2a4

Branch: refs/heads/branch-1
Commit: 2ee7d2a4882b18001bd5fbe7e5db32d6f75153ca
Parents: fa182c2
Author: Apekshit Sharma 
Authored: Fri Jun 3 16:34:52 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 16:35:29 2016 -0700

--
 bin/hirb.rb |  6 +-
 hbase-shell/src/main/ruby/hbase/admin.rb| 82 
 hbase-shell/src/main/ruby/hbase/hbase.rb| 34 
 hbase-shell/src/main/ruby/hbase/quotas.rb   |  3 +-
 .../src/main/ruby/hbase/replication_admin.rb|  3 +-
 hbase-shell/src/main/ruby/hbase/security.rb |  3 +-
 hbase-shell/src/main/ruby/hbase/table.rb|  2 +-
 hbase-shell/src/main/ruby/hbase/taskmonitor.rb  |  3 +-
 .../src/main/ruby/hbase/visibility_labels.rb|  5 +-
 hbase-shell/src/main/ruby/shell.rb  | 23 +++---
 hbase-shell/src/main/ruby/shell/commands.rb |  9 ++-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   |  1 -
 hbase-shell/src/test/ruby/hbase/hbase_test.rb   |  9 +--
 .../test/ruby/hbase/replication_admin_test.rb   |  1 -
 .../src/test/ruby/hbase/security_admin_test.rb  |  1 -
 .../ruby/hbase/visibility_labels_admin_test.rb  |  1 -
 .../src/test/ruby/shell/noninteractive_test.rb  |  4 +-
 hbase-shell/src/test/ruby/shell/shell_test.rb   |  4 +-
 hbase-shell/src/test/ruby/test_helper.rb|  8 +-
 19 files changed, 86 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2ee7d2a4/bin/hirb.rb
--
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 94b5cdb..2726e48 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -123,15 +123,11 @@ require 'shell'
 # Require formatter
 require 'shell/formatter'
 
-# Presume console format.
-# Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new
-
 # Setup the HBase module.  Create a configuration.
 @hbase = Hbase::Hbase.new
 
 # Setup console
-@shell = Shell::Shell.new(@hbase, @formatter, interactive)
+@shell = Shell::Shell.new(@hbase, interactive)
 @shell.debug = @shell_debug
 
 # Add commands to this namespace

http://git-wip-us.apache.org/repos/asf/hbase/blob/2ee7d2a4/hbase-shell/src/main/ruby/hbase/admin.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb 
b/hbase-shell/src/main/ruby/hbase/admin.rb
index f2bca0e..21e0652 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -32,10 +32,10 @@ module Hbase
   class Admin
 include HBaseConstants
 
-def initialize(admin, formatter)
-  @admin = admin
-  @connection = @admin.getConnection()
-  @formatter = formatter
+def initialize(connection)
+  @connection = connection
+  # Java Admin instance
+  @admin = @connection.getAdmin
 end
 
 def close
@@ -276,12 +276,6 @@ module Hbase
 end
 
 
#--
-# Parse arguments and update HTableDescriptor accordingly
-def parse_htd_args(htd, arg)
-  
htd.setNormalizationEnabled(JBoolean.valueOf(arg.delete(NORMALIZATION_ENABLED)))
 if arg[NORMALIZATION_ENABLED]
-end
-
-
#--
 # Creates a table
 def create(table_name, *args)
   # Fail if table name is not a string
@@ -359,24 +353,7 @@ module Hbase
 end
 
 # Done with splits; apply formerly-table_att parameters.
-htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER]
-htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if 
arg[MAX_FILESIZE]
-htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if 
arg[READONLY]
-htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if 
arg[COMPACTION_ENABLED]
-
htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if 
arg[MEMSTORE_FLUSHSIZE]
-# DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY.  To 
keep backward compatible, it still exists.
-#

hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

2016-06-03 Thread syuanjiang 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 1dfde7587 -> 73a746239


HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73a74623
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73a74623
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73a74623

Branch: refs/heads/branch-1.1
Commit: 73a7462395f7ece990bf07ea98c5772de87df441
Parents: 1dfde75
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:59:33 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/73a74623/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

2016-06-03 Thread syuanjiang 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 b41bf10ed -> edbf2754a


HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/edbf2754
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/edbf2754
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/edbf2754

Branch: refs/heads/branch-1.2
Commit: edbf2754a410d3a4496ba1c2c8f96c7919da50a8
Parents: b41bf10
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:57:11 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/edbf2754/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

2016-06-03 Thread syuanjiang 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 a2f9126ca -> aa636bef4


HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aa636bef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aa636bef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aa636bef

Branch: refs/heads/branch-1.3
Commit: aa636bef4ceb4254467faff8b4841275486e86bf
Parents: a2f9126
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:52:47 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/aa636bef/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

2016-06-03 Thread syuanjiang 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 9f343b587 -> fa182c2ea


HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa182c2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa182c2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa182c2e

Branch: refs/heads/branch-1
Commit: fa182c2ea68fcadb25beb78f4e99d55c0a507f36
Parents: 9f343b5
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:50:13 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fa182c2e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/master de1b5ff77 -> 70762faa9


HBASE-15849 Simplify the way we handle runtime of commands.
Functions format_simple_command and format_and_return_simple_command are used 
to print runtimes right now. They are called from within every single command 
and use Ruby's 'yield' magic.  Instead, we can simplify it using 'command_safe' 
function. Since command_safe wraps all commands, we can simply time before and 
after we call individual command.
If a command only wants to time a part of its logic, it can set instance 
variables start_time and end_time accordingly which is far more simpler to 
understand and work with than 'yield'.

Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70762faa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70762faa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70762faa

Branch: refs/heads/master
Commit: 70762faa98e6ad26f10fa8bd9ebc1824fdcb507c
Parents: de1b5ff
Author: Apekshit Sharma 
Authored: Fri Jun 3 14:13:45 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 15:50:03 2016 -0700

--
 hbase-shell/src/main/ruby/shell.rb  |  7 ++---
 hbase-shell/src/main/ruby/shell/commands.rb | 27 +++-
 .../main/ruby/shell/commands/abort_procedure.rb |  6 +
 .../src/main/ruby/shell/commands/add_labels.rb  |  4 +--
 .../src/main/ruby/shell/commands/add_peer.rb|  4 +--
 .../src/main/ruby/shell/commands/alter.rb   |  4 +--
 .../src/main/ruby/shell/commands/alter_async.rb |  4 +--
 .../main/ruby/shell/commands/alter_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/append.rb  | 10 
 .../ruby/shell/commands/append_peer_tableCFs.rb |  4 +--
 .../src/main/ruby/shell/commands/assign.rb  |  4 +--
 .../main/ruby/shell/commands/balance_switch.rb  |  6 +
 .../src/main/ruby/shell/commands/balancer.rb| 14 +-
 .../ruby/shell/commands/balancer_enabled.rb |  6 +
 .../shell/commands/catalogjanitor_enabled.rb|  6 +
 .../ruby/shell/commands/catalogjanitor_run.rb   |  4 +--
 .../shell/commands/catalogjanitor_switch.rb |  6 +
 .../src/main/ruby/shell/commands/clear_auths.rb |  4 +--
 .../main/ruby/shell/commands/clone_snapshot.rb  |  4 +--
 .../main/ruby/shell/commands/close_region.rb|  4 +--
 .../src/main/ruby/shell/commands/compact.rb |  4 +--
 .../src/main/ruby/shell/commands/compact_rs.rb  |  4 +--
 .../src/main/ruby/shell/commands/count.rb   |  4 +--
 .../src/main/ruby/shell/commands/create.rb  |  7 +++--
 .../ruby/shell/commands/create_namespace.rb |  4 +--
 .../src/main/ruby/shell/commands/delete.rb  | 13 +-
 .../ruby/shell/commands/delete_all_snapshot.rb  |  6 ++---
 .../main/ruby/shell/commands/delete_snapshot.rb |  4 +--
 .../shell/commands/delete_table_snapshots.rb| 19 +++---
 .../src/main/ruby/shell/commands/deleteall.rb   |  5 ++--
 .../src/main/ruby/shell/commands/describe.rb|  4 +--
 .../ruby/shell/commands/describe_namespace.rb   |  3 ---
 .../src/main/ruby/shell/commands/disable.rb |  4 +--
 .../main/ruby/shell/commands/disable_peer.rb|  4 +--
 .../shell/commands/disable_table_replication.rb |  6 ++---
 .../src/main/ruby/shell/commands/drop.rb|  4 +--
 .../main/ruby/shell/commands/drop_namespace.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable.rb  |  4 +--
 .../src/main/ruby/shell/commands/enable_peer.rb |  4 +--
 .../shell/commands/enable_table_replication.rb  |  6 ++---
 .../src/main/ruby/shell/commands/exists.rb  |  4 +--
 .../src/main/ruby/shell/commands/flush.rb   |  4 +--
 hbase-shell/src/main/ruby/shell/commands/get.rb |  4 +--
 .../src/main/ruby/shell/commands/get_auths.rb   |  8 +++---
 .../main/ruby/shell/commands/get_peer_config.rb |  7 +++--
 .../src/main/ruby/shell/commands/get_table.rb   |  4 +--
 .../src/main/ruby/shell/commands/grant.rb   |  9 +++
 .../src/main/ruby/shell/commands/incr.rb| 12 -
 .../src/main/ruby/shell/commands/is_disabled.rb |  8 ++
 .../src/main/ruby/shell/commands/is_enabled.rb  |  6 +
 .../src/main/ruby/shell/commands/list.rb|  3 +--
 .../src/main/ruby/shell/commands/list_labels.rb |  8 +++---
 .../main/ruby/shell/commands/list_namespace.rb  |  3 +--
 .../shell/commands/list_namespace_tables.rb |  3 +--
 .../ruby/shell/commands/list_peer_configs.rb| 18 ++---
 .../src/main/ruby/shell/commands/list_peers.rb  |  3 +--
 .../main/ruby/shell/commands/list_procedures.rb |  3 +--
 .../src/main/ruby/shell/commands/list_quotas.rb |  3 +--
 .../shell/commands/list_replicated_tables.rb|  4 +--
 .../main/ruby/shell/commands/list_snapshots.rb  |  3 +--
 .../ruby/shell/commands/list_table_snapshots.rb |  3 +--

hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)

2016-06-03 Thread syuanjiang 
Repository: hbase
Updated Branches:
  refs/heads/master 21e98271c -> de1b5ff77


HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active 
cleanup scan to finish (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de1b5ff7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de1b5ff7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de1b5ff7

Branch: refs/heads/master
Commit: de1b5ff7761b899691085e147c885e15897a562c
Parents: 21e9827
Author: Stephen Yuan Jiang 
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Fri Jun 3 15:49:23 2016 -0700

--
 .../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/de1b5ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index b9abc65..c93b307 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Triple;
 
 /**
@@ -86,7 +87,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
   public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an 
active
+// scan still going on, callers might not be aware of that and do further 
action thinkng
+// that no action would be from this chore.  In this case, the right 
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+  while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+  }
+}
+return alreadyEnabled;
   }
 
   boolean getEnabled() {

hbase git commit: HBASE-15881 Allow BZIP2 compression.

2016-06-03 Thread larsh 
Repository: hbase
Updated Branches:
  refs/heads/0.98 67dc17240 -> 804f8e7fd


HBASE-15881 Allow BZIP2 compression.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/804f8e7f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/804f8e7f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/804f8e7f

Branch: refs/heads/0.98
Commit: 804f8e7fd41e7872e0080cc3dd884c70c1fe29e9
Parents: 67dc172
Author: Lars Hofhansl 
Authored: Fri Jun 3 15:37:47 2016 -0700
Committer: Lars Hofhansl 
Committed: Fri Jun 3 15:37:47 2016 -0700

--
 .../hadoop/hbase/io/compress/Compression.java   | 29 +++-
 .../hadoop/hbase/util/TestCompressionTest.java  |  2 ++
 2 files changed, 30 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/804f8e7f/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index be7e5f6..edb0f48 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
   throw new RuntimeException(e);
 }
   }
-  };
+},
+BZIP2("bzip2") {
+  // Use base type to avoid compile-time dependencies.
+  private volatile transient CompressionCodec bzipCodec;
+  private transient Object lock = new Object();
+
+  @Override
+  CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+  synchronized (lock) {
+if (bzipCodec == null) {
+  bzipCodec = buildCodec(conf);
+}
+  }
+}
+return bzipCodec;
+  }
+
+  private CompressionCodec buildCodec(Configuration conf) {
+try {
+  Class externalCodec =
+  
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+  return (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
conf);
+} catch (ClassNotFoundException e) {
+  throw new RuntimeException(e);
+}
+  }
+};
 
 private final Configuration conf;
 private final String compressName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/804f8e7f/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index d5f3442..6ce8318 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
   nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
   nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
   nativeCodecTest("SNAPPY", "snappy", 
"org.apache.hadoop.io.compress.SnappyCodec");
+  nativeCodecTest("BZIP2", "bzip2", 
"org.apache.hadoop.io.compress.BZip2Codec");
 } else {
   // Hadoop nativelib is not available
   LOG.debug("Native code not loaded");
   assertFalse(CompressionTest.testCompression("LZO"));
   assertFalse(CompressionTest.testCompression("LZ4"));
   assertFalse(CompressionTest.testCompression("SNAPPY"));
+  assertFalse(CompressionTest.testCompression("BZIP2"));
 }
   }

[2/3] hbase git commit: HBASE-15881 Allow BZIP2 compression.

2016-06-03 Thread larsh 
HBASE-15881 Allow BZIP2 compression.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f343b58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f343b58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f343b58

Branch: refs/heads/branch-1
Commit: 9f343b5871033e1be166e6ab605189c89bbcf7d5
Parents: 7bcdd44
Author: Lars Hofhansl 
Authored: Thu Jun 2 12:19:02 2016 -0700
Committer: Lars Hofhansl 
Committed: Fri Jun 3 15:33:05 2016 -0700

--
 .../hadoop/hbase/io/compress/Compression.java   | 31 ++--
 .../hadoop/hbase/util/TestCompressionTest.java  |  2 ++
 2 files changed, 31 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9f343b58/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index c6ebff5..099b445 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
   throw new RuntimeException(e);
 }
   }
-  };
+},
+BZIP2("bzip2") {
+  // Use base type to avoid compile-time dependencies.
+  private volatile transient CompressionCodec bzipCodec;
+  private transient Object lock = new Object();
+
+  @Override
+  CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+  synchronized (lock) {
+if (bzipCodec == null) {
+  bzipCodec = buildCodec(conf);
+}
+  }
+}
+return bzipCodec;
+  }
+
+  private CompressionCodec buildCodec(Configuration conf) {
+try {
+  Class externalCodec =
+  
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+  return (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
conf);
+} catch (ClassNotFoundException e) {
+  throw new RuntimeException(e);
+}
+  }
+};
 
 private final transient Configuration conf; // FindBugs: SE_BAD_FIELD so 
just made it transient
 private final String compressName;
@@ -431,4 +458,4 @@ public final class Compression {
   }
 }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f343b58/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index 682b066..b8e79ed 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
   nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
   nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
   nativeCodecTest("SNAPPY", "snappy", 
"org.apache.hadoop.io.compress.SnappyCodec");
+  nativeCodecTest("BZIP2", "bzip2", 
"org.apache.hadoop.io.compress.BZip2Codec");
 } else {
   // Hadoop nativelib is not available
   LOG.debug("Native code not loaded");
   assertFalse(CompressionTest.testCompression("LZO"));
   assertFalse(CompressionTest.testCompression("LZ4"));
   assertFalse(CompressionTest.testCompression("SNAPPY"));
+  assertFalse(CompressionTest.testCompression("BZIP2"));
 }
   }

[3/3] hbase git commit: HBASE-15881 Allow BZIP2 compression.

2016-06-03 Thread larsh 
HBASE-15881 Allow BZIP2 compression.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2f9126c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2f9126c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2f9126c

Branch: refs/heads/branch-1.3
Commit: a2f9126cad196c6e305b15615b8165fda8a47914
Parents: b0e1fda
Author: Lars Hofhansl 
Authored: Thu Jun 2 12:19:02 2016 -0700
Committer: Lars Hofhansl 
Committed: Fri Jun 3 15:33:31 2016 -0700

--
 .../hadoop/hbase/io/compress/Compression.java   | 31 ++--
 .../hadoop/hbase/util/TestCompressionTest.java  |  2 ++
 2 files changed, 31 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a2f9126c/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index c6ebff5..099b445 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
   throw new RuntimeException(e);
 }
   }
-  };
+},
+BZIP2("bzip2") {
+  // Use base type to avoid compile-time dependencies.
+  private volatile transient CompressionCodec bzipCodec;
+  private transient Object lock = new Object();
+
+  @Override
+  CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+  synchronized (lock) {
+if (bzipCodec == null) {
+  bzipCodec = buildCodec(conf);
+}
+  }
+}
+return bzipCodec;
+  }
+
+  private CompressionCodec buildCodec(Configuration conf) {
+try {
+  Class externalCodec =
+  
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+  return (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
conf);
+} catch (ClassNotFoundException e) {
+  throw new RuntimeException(e);
+}
+  }
+};
 
 private final transient Configuration conf; // FindBugs: SE_BAD_FIELD so 
just made it transient
 private final String compressName;
@@ -431,4 +458,4 @@ public final class Compression {
   }
 }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/a2f9126c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index 682b066..b8e79ed 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
   nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
   nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
   nativeCodecTest("SNAPPY", "snappy", 
"org.apache.hadoop.io.compress.SnappyCodec");
+  nativeCodecTest("BZIP2", "bzip2", 
"org.apache.hadoop.io.compress.BZip2Codec");
 } else {
   // Hadoop nativelib is not available
   LOG.debug("Native code not loaded");
   assertFalse(CompressionTest.testCompression("LZO"));
   assertFalse(CompressionTest.testCompression("LZ4"));
   assertFalse(CompressionTest.testCompression("SNAPPY"));
+  assertFalse(CompressionTest.testCompression("BZIP2"));
 }
   }

[1/3] hbase git commit: HBASE-14307 - Incorrect use of positional read api in HFileBlock (Chris Nauroth)

2016-06-03 Thread larsh 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 7bcdd44be -> 9f343b587
  refs/heads/branch-1.3 b0e1fdae3 -> a2f9126ca
  refs/heads/branch-2 [created] 75bb6d96b


HBASE-14307 - Incorrect use of positional read api in HFileBlock (Chris
Nauroth)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75bb6d96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75bb6d96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75bb6d96

Branch: refs/heads/branch-2
Commit: 75bb6d96bcf974dd04da3543638dd1bf7d9a6156
Parents: 3703ad4
Author: ramkrishna 
Authored: Fri Sep 11 15:56:09 2015 +0530
Committer: ramkrishna 
Committed: Fri Sep 11 16:01:56 2015 +0530

--
 .../hadoop/hbase/io/hfile/HFileBlock.java   |  50 ++-
 .../io/hfile/TestHFileBlockPositionalRead.java  | 148 +++
 2 files changed, 190 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/75bb6d96/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 1415a00..2da91b6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.util.ChecksumType;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.IOUtils;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
@@ -695,6 +696,45 @@ public class HFileBlock implements Cacheable {
   }
 
   /**
+   * Read from an input stream. Analogous to
+   * {@link IOUtils#readFully(InputStream, byte[], int, int)}, but uses
+   * positional read and specifies a number of "extra" bytes that would be
+   * desirable but not absolutely necessary to read.
+   *
+   * @param in the input stream to read from
+   * @param position the position within the stream from which to start reading
+   * @param buf the buffer to read into
+   * @param bufOffset the destination offset in the buffer
+   * @param necessaryLen the number of bytes that are absolutely necessary to
+   * read
+   * @param extraLen the number of extra bytes that would be nice to read
+   * @return true if and only if extraLen is > 0 and reading those extra bytes
+   * was successful
+   * @throws IOException if failed to read the necessary bytes
+   */
+  @VisibleForTesting
+  static boolean positionalReadWithExtra(FSDataInputStream in,
+  long position, byte[] buf, int bufOffset, int necessaryLen, int extraLen)
+  throws IOException {
+int bytesRemaining = necessaryLen + extraLen;
+int bytesRead = 0;
+while (bytesRead < necessaryLen) {
+  int ret = in.read(position, buf, bufOffset, bytesRemaining);
+  if (ret < 0) {
+throw new IOException("Premature EOF from inputStream (positional read 
"
++ "returned " + ret + ", was trying to read " + necessaryLen
++ " necessary bytes and " + extraLen + " extra bytes, "
++ "successfully read " + bytesRead);
+  }
+  position += ret;
+  bufOffset += ret;
+  bytesRemaining -= ret;
+  bytesRead += ret;
+}
+return bytesRead != necessaryLen && bytesRemaining <= 0;
+  }
+
+  /**
* @return the on-disk size of the next block (including the header size)
* that was read by peeking into the next block's header
*/
@@ -1377,14 +1417,8 @@ public class HFileBlock implements Cacheable {
   } else {
 // Positional read. Better for random reads; or when the streamLock is 
already locked.
 int extraSize = peekIntoNextBlock ? hdrSize : 0;
-int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
-if (ret < size) {
-  throw new IOException("Positional read of " + size + " bytes " +
-  "failed at offset " + fileOffset + " (returned " + ret + ")");
-}
-
-if (ret == size || ret < size + extraSize) {
-  // Could not read the next block's header, or did not try.
+if (!positionalReadWithExtra(istream, fileOffset, dest, destOffset,
+size, extraSize)) {
   return -1;
 }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/75bb6d96/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockPositionalRead.java
--
diff --git

hbase git commit: HBASE-15883 Adding WAL files and tracking offsets in HBase.

2016-06-03 Thread eclark 
Repository: hbase
Updated Branches:
  refs/heads/master 9a53d8b38 -> 21e98271c


HBASE-15883 Adding WAL files and tracking offsets in HBase.

Implemented ReplicationQueuesHBaseImpl that tracks WAL offsets and replication 
queues in an HBase table.
Only wrote the basic tracking methods, have not implemented claimQueue() or 
HFileRef methods yet.
Wrote a basic unit test for ReplicationQueueHBaseImpl that tests the 
implemented functions on a single Region Server

Signed-off-by: Elliott Clark 
Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21e98271
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21e98271
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21e98271

Branch: refs/heads/master
Commit: 21e98271c32f0d44106515a72b2c92d518c03668
Parents: 9a53d8b
Author: Joseph Hwang 
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 3 15:23:10 2016 -0700

--
 .../hbase/replication/ReplicationFactory.java   |  11 +-
 .../hbase/replication/ReplicationQueues.java|   8 +-
 .../replication/ReplicationQueuesArguments.java |  66 +++
 .../replication/ReplicationQueuesHBaseImpl.java | 491 +++
 .../replication/ReplicationQueuesZKImpl.java|  13 +-
 .../replication/regionserver/Replication.java   |  12 +-
 .../regionserver/ReplicationSourceManager.java  |   5 +-
 .../replication/TestReplicationAdmin.java   |   3 +-
 .../hbase/master/cleaner/TestLogsCleaner.java   |   3 +-
 .../cleaner/TestReplicationHFileCleaner.java|   4 +-
 .../replication/TestReplicationStateBasic.java  |   2 +-
 .../TestReplicationStateHBaseImpl.java  | 243 +
 .../replication/TestReplicationStateZKImpl.java |  13 +-
 .../TestReplicationSourceManager.java   |  36 +-
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |   4 +-
 15 files changed, 871 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 91e77ca..e264a4d 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import org.apache.commons.lang.reflect.ConstructorUtils;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
@@ -30,9 +31,11 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 @InterfaceAudience.Private
 public class ReplicationFactory {
 
-  public static ReplicationQueues getReplicationQueues(final ZooKeeperWatcher 
zk,
-  Configuration conf, Abortable abortable) {
-return new ReplicationQueuesZKImpl(zk, conf, abortable);
+  public static ReplicationQueues 
getReplicationQueues(ReplicationQueuesArguments args)
+  throws Exception {
+Class classToBuild = args.getConf().getClass("hbase.region.replica." +
+"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class);
+return (ReplicationQueues) 
ConstructorUtils.invokeConstructor(classToBuild, args);
   }
 
   public static ReplicationQueuesClient getReplicationQueuesClient(final 
ZooKeeperWatcher zk,
@@ -44,7 +47,7 @@ public class ReplicationFactory {
   Abortable abortable) {
 return getReplicationPeers(zk, conf, null, abortable);
   }
-  
+
   public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher 
zk, Configuration conf,
   final ReplicationQueuesClient queuesClient, Abortable abortable) {
 return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);

http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0d47a88..db6da91 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -83,13 +83,13 @@ public interface ReplicationQueues {
   /**
* Get a list of all

hbase git commit: HBASE-15929 There are two tests named TestRegionServerMetrics. This causes slight issues when parsing console output for hanging tests and flaky tests analysis. Moving .../hbase/Test

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/master b557f0bec -> 9a53d8b38


HBASE-15929 There are two tests named TestRegionServerMetrics. This causes 
slight issues when parsing console output for hanging tests and flaky tests 
analysis. Moving .../hbase/TestRegionServerMetrics.java to 
.../hbase/regionserver/TestRegionServerReadRequestMetrics.java. (Apekshit)

Change-Id: I379c15fe2c2c01bed53bddf7619d5f2a07c5640e


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a53d8b3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a53d8b3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a53d8b3

Branch: refs/heads/master
Commit: 9a53d8b3850f0521c3a4ad194a3b84d7f4145007
Parents: b557f0b
Author: Apekshit 
Authored: Thu Jun 2 20:06:20 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 15:13:49 2016 -0700

--
 .../hadoop/hbase/TestRegionServerMetrics.java   | 379 --
 .../TestRegionServerReadRequestMetrics.java | 387 +++
 2 files changed, 387 insertions(+), 379 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a53d8b3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
deleted file mode 100644
index 76e5842..000
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-@Category(MediumTests.class)
-public class TestRegionServerMetrics {
-  private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
-  private static final TableName TABLE_NAME = TableName.valueOf("test");
-  private static final byte[] CF1 = "c1".getBytes();
-  private static final byte[] CF2 = "c2".getBytes();
-
-  private static final byte[] ROW1 = "a".getBytes();
-  private static final byte[] ROW2 = "b".getBytes();
-  private static final byte[] ROW3 = "c".getBytes();
-  private static final byte[] COL1 = "q1".getBytes();
-  private static final byte[] COL2 = "q2".getBytes();
-  private static final byte[] COL3 = "q3".getBytes();
-  private static final byte[] VAL1 = "v1".getBytes();
-  private static final byte[] VAL2 = "v2".getBytes();
-  private static final byte[] VAL3 = Bytes.toBytes(0L);
-
-  private static final int MAX_TRY = 20;
-  private static final int SLEEP_MS = 100;
-  private static final int TTL = 1;
-
-  private static Admin admin;
-  private static Collection serverNames;
-  private static Table table;
-  private static List tableRegions;
-
-

hbase git commit: HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to s

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/master bdb46f01b -> b557f0bec


HBASE-15949 Cleanup TestRegionServerMetrics.
@Before and @After to setup/teardown tables using @Rule to set table name based 
on testname.
Refactor out copy-pasted code fragments to single function.
(Apekshit)

Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b557f0be
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b557f0be
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b557f0be

Branch: refs/heads/master
Commit: b557f0bec62a48753e5d01d7a47f3c9e5a6b3ee8
Parents: bdb46f0
Author: Apekshit 
Authored: Wed Jun 1 21:52:29 2016 -0700
Committer: Apekshit Sharma 
Committed: Fri Jun 3 14:27:24 2016 -0700

--
 .../regionserver/TestRegionServerMetrics.java   | 682 +++
 1 file changed, 256 insertions(+), 426 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b557f0be/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 18796bd..06db468 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.*;
@@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
 
 import static org.junit.Assert.*;
 
@@ -39,23 +47,35 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-
 @Category({RegionServerTests.class, MediumTests.class})
 public class TestRegionServerMetrics {
-  private static MetricsAssertHelper metricsHelper;
+  private static final Log LOG = 
LogFactory.getLog(TestRegionServerMetrics.class);
+
+  @Rule
+  public TestName testName = new TestName();
+
+  @ClassRule
+  public static TestRule timeout = 
CategoryBasedTimeout.forClass(TestRegionServerMetrics.class);
 
   static {
 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
   }
 
+  private static MetricsAssertHelper metricsHelper;
   private static MiniHBaseCluster cluster;
   private static HRegionServer rs;
   private static Configuration conf;
   private static HBaseTestingUtility TEST_UTIL;
+  private static Connection connection;
   private static MetricsRegionServer metricsRegionServer;
   private static MetricsRegionServerSource serverSource;
   private static final int NUM_SCAN_NEXT = 30;
   private static int numScanNext = 0;
+  private static byte[] cf = Bytes.toBytes("cf");
+  private static byte[] row = Bytes.toBytes("row");
+  private static byte[] qualifier = Bytes.toBytes("qual");
+  private static byte[] val = Bytes.toBytes("val");
+  private static Admin admin;
 
   @BeforeClass
   public static void startCluster() throws Exception {
@@ -65,12 +85,16 @@ public class TestRegionServerMetrics {
 conf.getLong("hbase.splitlog.max.resubmit", 0);
 // Make the failure test faster
 conf.setInt("zookeeper.recovery.retry", 0);
+// testMobMetrics creates few hfiles and manages compaction manually.
+conf.setInt("hbase.hstore.compactionThreshold", 100);
+conf.setInt("hbase.hstore.compaction.max", 100);
 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
 
 TEST_UTIL.startMiniCluster(1, 1);
 cluster = TEST_UTIL.getHBaseCluster();
-
 cluster.waitForActiveAndReadyMaster();
+admin = TEST_UTIL.getHBaseAdmin();
+connection = TEST_UTIL.getConnection();
 
 while (cluster.getLiveRegionServerThreads().size() < 1) {
   Threads.sleep(100);
@@ -88,551 +112,370 @@ public class TestRegionServerMetrics {
 }
   }
 
-  @Test(timeout = 30)
+  TableName tableName;
+  Table table;
+
+  @Before
+  public void beforeTestMethod() throws Exception {
+

hbase git commit: HBASE-15845 Changes: - Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files right now which is confusing. - Remove omnipresence of formatter object since it i

2016-06-03 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/master f0c159b5f -> bdb46f01b


HBASE-15845 Changes:
- Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files 
right now which is confusing.
- Remove omnipresence of formatter object since it is kind of a use-and-throw 
class. Commands should create
  an instance, use it to format the output and discard it.
- Some refactoring

Change-Id: If9ea9873904e0a39d199a6aa10e23864b86a2f09


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdb46f01
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdb46f01
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdb46f01

Branch: refs/heads/master
Commit: bdb46f01b9645a0aa4e9827a7e627de76cd7983b
Parents: f0c159b
Author: Apekshit 
Authored: Mon Dec 28 14:50:50 2015 -0800
Committer: Apekshit Sharma 
Committed: Fri Jun 3 13:37:38 2016 -0700

--
 bin/hirb.rb |   8 +-
 hbase-shell/src/main/ruby/hbase.rb  | 109 ---
 hbase-shell/src/main/ruby/hbase/admin.rb|  82 ++
 hbase-shell/src/main/ruby/hbase/hbase.rb|  36 +++---
 hbase-shell/src/main/ruby/hbase/quotas.rb   |   3 +-
 .../src/main/ruby/hbase/replication_admin.rb|   3 +-
 .../src/main/ruby/hbase/rsgroup_admin.rb|   3 +-
 hbase-shell/src/main/ruby/hbase/security.rb |   3 +-
 hbase-shell/src/main/ruby/hbase/table.rb|   2 +-
 hbase-shell/src/main/ruby/hbase/taskmonitor.rb  |   3 +-
 .../src/main/ruby/hbase/visibility_labels.rb|   5 +-
 hbase-shell/src/main/ruby/hbase_constants.rb| 109 +++
 hbase-shell/src/main/ruby/shell.rb  |  25 ++---
 hbase-shell/src/main/ruby/shell/commands.rb |   9 +-
 hbase-shell/src/test/ruby/hbase/admin_test.rb   |   3 +-
 hbase-shell/src/test/ruby/hbase/hbase_test.rb   |  11 +-
 .../test/ruby/hbase/replication_admin_test.rb   |   3 +-
 .../src/test/ruby/hbase/security_admin_test.rb  |   3 +-
 hbase-shell/src/test/ruby/hbase/table_test.rb   |   2 +-
 .../src/test/ruby/hbase/taskmonitor_test.rb |   2 +-
 .../ruby/hbase/visibility_labels_admin_test.rb  |   3 +-
 .../src/test/ruby/shell/commands_test.rb|   2 +-
 .../src/test/ruby/shell/noninteractive_test.rb  |   6 +-
 .../src/test/ruby/shell/rsgroup_shell_test.rb   |  18 ++-
 hbase-shell/src/test/ruby/shell/shell_test.rb   |   6 +-
 hbase-shell/src/test/ruby/test_helper.rb|  10 +-
 26 files changed, 218 insertions(+), 251 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/bin/hirb.rb
--
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 94b5cdb..d0295d6 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -115,7 +115,7 @@ 
org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level)
 
org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
 
 # Require HBase now after setting log levels
-require 'hbase'
+require 'hbase_constants'
 
 # Load hbase shell
 require 'shell'
@@ -123,15 +123,11 @@ require 'shell'
 # Require formatter
 require 'shell/formatter'
 
-# Presume console format.
-# Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new
-
 # Setup the HBase module.  Create a configuration.
 @hbase = Hbase::Hbase.new
 
 # Setup console
-@shell = Shell::Shell.new(@hbase, @formatter, interactive)
+@shell = Shell::Shell.new(@hbase, interactive)
 @shell.debug = @shell_debug
 
 # Add commands to this namespace

http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/hbase-shell/src/main/ruby/hbase.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase.rb 
b/hbase-shell/src/main/ruby/hbase.rb
deleted file mode 100644
index bc6f37c..000
--- a/hbase-shell/src/main/ruby/hbase.rb
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# HBase ruby

[2/2] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()

2016-06-03 Thread mbertozzi 
HBASE-15927 Remove HMaster.assignRegion()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7bcdd44b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7bcdd44b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7bcdd44b

Branch: refs/heads/branch-1
Commit: 7bcdd44be6951aae3f3e041b682b7d06c06f1721
Parents: 950a09b
Author: Matteo Bertozzi 
Authored: Fri Jun 3 12:41:51 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 3 12:41:51 2016 -0700

--
 .../org/apache/hadoop/hbase/master/HMaster.java |  4 --
 .../hadoop/hbase/master/MasterRpcServices.java  |  4 +-
 .../hadoop/hbase/HBaseTestingUtility.java   | 15 ++
 .../hbase/client/TestMetaWithReplicas.java  |  2 +-
 .../master/TestAssignmentManagerOnCluster.java  | 50 
 .../hadoop/hbase/master/TestMasterFailover.java | 36 +++---
 .../apache/hadoop/hbase/util/TestHBaseFsck.java | 35 --
 7 files changed, 67 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index beb1bca..4b7e181 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2387,10 +2387,6 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
 return this.initializationBeforeMetaAssignment;
   }
 
-  public void assignRegion(HRegionInfo hri) {
-assignmentManager.assign(hri, true);
-  }
-
   /**
* Compute the average load across all region servers.
* Currently, this uses a very naive computation - just uses the number of

http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 90471cf..152379d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1389,7 +1389,7 @@ public class MasterRpcServices extends RSRpcServices
   if (master.assignmentManager.getRegionStates().isRegionOffline(hri)) {
 LOG.debug("Region " + hri.getRegionNameAsString()
 + " is not online on any region server, reassigning it.");
-master.assignRegion(hri);
+master.assignmentManager.assign(hri, true);
   }
   if (master.cpHost != null) {
 master.cpHost.postUnassign(hri, force);
@@ -1572,7 +1572,7 @@ public class MasterRpcServices extends RSRpcServices
 }
   }
 
-  /** 
+  /**
* Returns the security capabilities in effect on the cluster
*/
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 2870593..4547e0f 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3388,6 +3388,21 @@ public class HBaseTestingUtility extends 
HBaseCommonTestingUtility {
   }
 
   /**
+   * Uses directly the assignment manager to assign the region.
+   * and waits until the specified region has completed assignment.
+   * @param tableName the table name
+   * @throws IOException
+   * @throw InterruptedException
+   * @return true if the region is assigned false otherwise.
+   */
+  public boolean assignRegion(final HRegionInfo regionInfo)
+  throws IOException, InterruptedException {
+final AssignmentManager am = 
getHBaseCluster().getMaster().getAssignmentManager();
+am.assign(regionInfo, true);
+return am.waitForAssignment(regionInfo);
+  }
+
+  /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,

[1/2] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()

2016-06-03 Thread mbertozzi 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 950a09b03 -> 7bcdd44be
  refs/heads/master 0cbce0762 -> f0c159b5f


HBASE-15927 Remove HMaster.assignRegion()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0c159b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0c159b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0c159b5

Branch: refs/heads/master
Commit: f0c159b5fed732cdc08bd2fd924ae81677d66a30
Parents: 0cbce07
Author: Matteo Bertozzi 
Authored: Fri Jun 3 12:36:25 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Jun 3 12:36:25 2016 -0700

--
 .../org/apache/hadoop/hbase/master/HMaster.java |  4 --
 .../hadoop/hbase/HBaseTestingUtility.java   | 15 +
 .../hbase/client/TestMetaWithReplicas.java  |  2 +-
 .../master/TestAssignmentManagerOnCluster.java  | 61 
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   | 20 +++
 .../hadoop/hbase/util/TestHBaseFsckTwoRS.java   | 23 +++-
 6 files changed, 58 insertions(+), 67 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f8d0003..d368ffb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2378,10 +2378,6 @@ public class HMaster extends HRegionServer implements 
MasterServices {
 return this.initializationBeforeMetaAssignment;
   }
 
-  public void assignRegion(HRegionInfo hri) {
-assignmentManager.assign(hri);
-  }
-
   /**
* Compute the average load across all region servers.
* Currently, this uses a very naive computation - just uses the number of

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 7b35815..bfa14cb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3200,6 +3200,21 @@ public class HBaseTestingUtility extends 
HBaseCommonTestingUtility {
   }
 
   /**
+   * Uses directly the assignment manager to assign the region.
+   * and waits until the specified region has completed assignment.
+   * @param tableName the table name
+   * @throws IOException
+   * @throw InterruptedException
+   * @return true if the region is assigned false otherwise.
+   */
+  public boolean assignRegion(final HRegionInfo regionInfo)
+  throws IOException, InterruptedException {
+final AssignmentManager am = 
getHBaseCluster().getMaster().getAssignmentManager();
+am.assign(regionInfo);
+return am.waitForAssignment(regionInfo);
+  }
+
+  /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index 5e302d2..8e87ceb 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -448,7 +448,7 @@ public class TestMetaWithReplicas {
 // create in-memory state otherwise master won't assign
 TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
  .getRegionStates().createRegionState(h);
-TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h);
+TEST_UTIL.assignRegion(h);
 HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h);
 // check that problem exists
 HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java

hbase git commit: HBASE-15953 HBase backup set list command does not list any existing sets (Vladimir)

2016-06-03 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 c51700180 -> 964c0841d


HBASE-15953 HBase backup set list command does not list any existing sets 
(Vladimir)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/964c0841
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/964c0841
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/964c0841

Branch: refs/heads/HBASE-7912
Commit: 964c0841db67464fc5b96807dfd321b44de8b5d7
Parents: c517001
Author: tedyu 
Authored: Fri Jun 3 09:31:13 2016 -0700
Committer: tedyu 
Committed: Fri Jun 3 09:31:13 2016 -0700

--
 .../org/apache/hadoop/hbase/backup/impl/BackupCommands.java | 5 -
 .../java/org/apache/hadoop/hbase/backup/TestBackupBase.java | 9 -
 2 files changed, 4 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/964c0841/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 1fce171..9a8697a 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -466,7 +466,10 @@ public final class BackupCommands {
   Configuration conf = getConf() != null? getConf(): 
HBaseConfiguration.create();
   try(final Connection conn = ConnectionFactory.createConnection(conf); 
   final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
-admin.listBackupSets();
+List list = admin.listBackupSets();
+for(BackupSet bs: list){
+  System.out.println(bs);
+}
   }
 }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/964c0841/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
index 9683e16..31cf65a 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
@@ -249,15 +249,6 @@ public class TestBackupBase {
 }
   }
 
-//  protected BackupClient getBackupClient(){
-//return BackupRestoreClientFactory.getBackupClient(conf1);
-//  }
-  
-//  protected RestoreClient getRestoreClient()
-//  {
-//return BackupRestoreClientFactory.getRestoreClient(conf1);
-//  }
-
   protected BackupAdmin getBackupAdmin() throws IOException {
 return TEST_UTIL.getAdmin().getBackupAdmin();
   }

hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication

2016-06-03 Thread ashishsinghi 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 5824f2236 -> b0e1fdae3


HBASE-15888 Extend HBASE-12769 for bulk load data replication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0e1fdae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0e1fdae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0e1fdae

Branch: refs/heads/branch-1.3
Commit: b0e1fdae346b64af4188cf5df29488617753416f
Parents: 5824f22
Author: Ashish Singhi 
Authored: Fri Jun 3 18:48:47 2016 +0530
Committer: Ashish Singhi 
Committed: Fri Jun 3 18:48:47 2016 +0530

--
 .../replication/ReplicationPeersZKImpl.java |  6 ++
 .../hbase/util/hbck/ReplicationChecker.java | 59 ++--
 2 files changed, 61 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e1fdae/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index ad634fa..b0d6e83 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -601,6 +601,12 @@ public class ReplicationPeersZKImpl extends 
ReplicationStateZKBase implements Re
   }
 }
   }
+  // Check for hfile-refs queue
+  if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+  && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+  }
 } catch (KeeperException e) {
   throw new ReplicationException("Could not check queues deleted with id=" 
+ peerId, e);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e1fdae/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index bf44a50..64212c9 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
 @InterfaceAudience.Private
 public class ReplicationChecker {
   private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+  private final ZooKeeperWatcher zkw;
   private ErrorReporter errorReporter;
   private ReplicationQueuesClient queuesClient;
   private ReplicationPeers replicationPeers;
   private ReplicationQueueDeletor queueDeletor;
   // replicator with its queueIds for removed peers
   private Map undeletedQueueIds = new HashMap();
-  
+  // Set of un deleted hfile refs queue Ids
+  private Set undeletedHFileRefsQueueIds = new HashSet<>();
+  private final String hfileRefsZNode;
+
   public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw, 
HConnection connection,
   ErrorReporter errorReporter) throws IOException {
 try {
+  this.zkw = zkw;
   this.errorReporter = errorReporter;
   this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw, 
conf, connection);
   this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
 } catch (ReplicationException e) {
   throw new IOException("failed to construct ReplicationChecker", e);
 }
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication", 
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode, 
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+  
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
   }
 
   public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
 } catch (KeeperException ke) {
   throw new IOException(ke);
 }
+
+checkUnDeletedHFileRefsQueues(peerIds);
+  }
+
+  private void checkUnDeletedHFileRefsQueues(Set peerIds) throws 
IOException {
+try {
+  if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+

hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication

2016-06-03 Thread ashishsinghi 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 a8c8bfd5e -> 950a09b03


HBASE-15888 Extend HBASE-12769 for bulk load data replication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/950a09b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/950a09b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/950a09b0

Branch: refs/heads/branch-1
Commit: 950a09b03cf1a306ed5a2754c11457daf30e7d23
Parents: a8c8bfd
Author: Ashish Singhi 
Authored: Fri Jun 3 18:46:41 2016 +0530
Committer: Ashish Singhi 
Committed: Fri Jun 3 18:47:31 2016 +0530

--
 .../replication/ReplicationPeersZKImpl.java |  6 ++
 .../hbase/util/hbck/ReplicationChecker.java | 59 ++--
 2 files changed, 61 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/950a09b0/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index c0c3f7e..076167e 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -567,6 +567,12 @@ public class ReplicationPeersZKImpl extends 
ReplicationStateZKBase implements Re
   }
 }
   }
+  // Check for hfile-refs queue
+  if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+  && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+  }
 } catch (KeeperException e) {
   throw new ReplicationException("Could not check queues deleted with id=" 
+ peerId, e);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/950a09b0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index bf44a50..64212c9 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
 @InterfaceAudience.Private
 public class ReplicationChecker {
   private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+  private final ZooKeeperWatcher zkw;
   private ErrorReporter errorReporter;
   private ReplicationQueuesClient queuesClient;
   private ReplicationPeers replicationPeers;
   private ReplicationQueueDeletor queueDeletor;
   // replicator with its queueIds for removed peers
   private Map undeletedQueueIds = new HashMap();
-  
+  // Set of un deleted hfile refs queue Ids
+  private Set undeletedHFileRefsQueueIds = new HashSet<>();
+  private final String hfileRefsZNode;
+
   public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw, 
HConnection connection,
   ErrorReporter errorReporter) throws IOException {
 try {
+  this.zkw = zkw;
   this.errorReporter = errorReporter;
   this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw, 
conf, connection);
   this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
 } catch (ReplicationException e) {
   throw new IOException("failed to construct ReplicationChecker", e);
 }
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication", 
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode, 
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+  
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
   }
 
   public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
 } catch (KeeperException ke) {
   throw new IOException(ke);
 }
+
+checkUnDeletedHFileRefsQueues(peerIds);
+  }
+
+  private void checkUnDeletedHFileRefsQueues(Set peerIds) throws 
IOException {
+try {
+  if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+