hbase git commit: Revert "HBASE-15949 Cleanup TestRegionServerMetrics." This change passes on master but fails on branch-1. Will try to debug.
Repository: hbase
Updated Branches:
refs/heads/branch-1 520d27b26 -> 6d0e8b2a9
Revert "HBASE-15949 Cleanup TestRegionServerMetrics."
This change passes on master but fails on branch-1. Will try to debug.
This reverts commit 520d27b267737e43718723692bfdb161139c4de6.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d0e8b2a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d0e8b2a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d0e8b2a
Branch: refs/heads/branch-1
Commit: 6d0e8b2a91bec047dcd4f66f38352aa09bdcc3f1
Parents: 520d27b
Author: Apekshit Sharma
Authored: Fri Jun 3 23:55:53 2016 -0700
Committer: Apekshit Sharma
Committed: Fri Jun 3 23:55:53 2016 -0700
--
.../regionserver/TestRegionServerMetrics.java | 596 ---
1 file changed, 387 insertions(+), 209 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d0e8b2a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index b7b72a6..d068217 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -17,29 +17,20 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.ClassRule;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
import static org.junit.Assert.*;
@@ -47,36 +38,23 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-@Category({RegionServerTests.class, MediumTests.class})
-public class TestRegionServerMetrics {
- private static final Log LOG =
LogFactory.getLog(TestRegionServerMetrics.class);
-
- @Rule
- public TestName testName = new TestName();
- @ClassRule
- public static TestRule timeout = CategoryBasedTimeout.builder()
-
.withTimeout(TestRegionServerMetrics.class).withLookingForStuckThread(true).build();
+@Category(MediumTests.class)
+public class TestRegionServerMetrics {
+ private static MetricsAssertHelper metricsHelper;
static {
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
}
- private static MetricsAssertHelper metricsHelper;
private static MiniHBaseCluster cluster;
private static HRegionServer rs;
private static Configuration conf;
private static HBaseTestingUtility TEST_UTIL;
- private static Connection connection;
private static MetricsRegionServer metricsRegionServer;
private static MetricsRegionServerSource serverSource;
private static final int NUM_SCAN_NEXT = 30;
private static int numScanNext = 0;
- private static byte[] cf = Bytes.toBytes("cf");
- private static byte[] row = Bytes.toBytes("row");
- private static byte[] qualifier = Bytes.toBytes("qual");
- private static byte[] val = Bytes.toBytes("val");
- private static Admin admin;
@BeforeClass
public static void startCluster() throws Exception {
@@ -86,16 +64,12 @@ public class TestRegionServerMetrics {
conf.getLong("hbase.splitlog.max.resubmit", 0);
// Make the failure test faster
conf.setInt("zookeeper.recovery.retry", 0);
-// testMobMetrics creates few hfiles and manages compaction manually.
-conf.setInt("hbase.hstore.compactionThreshold", 100);
-conf.setInt("hbase.hstore.compaction.max", 100);
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
TEST_UTIL.startMiniCluster(1, 1);
cluster = TEST_UTIL.getHBaseCluster();
+
cluster.waitForActiveAndReadyMaster();
-admin = TEST_UTIL.getHBaseAdmin();
-connection = TEST_UTIL.getConnection();
while (cluster.getLiveRegionServerThreads().size() < 1) {
Threads.sleep(100);
@@ -113,129 +87,85 @@ public cla
hbase git commit: HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to s
Repository: hbase
Updated Branches:
refs/heads/branch-1 1972996ff -> 520d27b26
HBASE-15949 Cleanup TestRegionServerMetrics.
@Before and @After to setup/teardown tables using @Rule to set table name based
on testname.
Refactor out copy-pasted code fragments to single function.
Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/520d27b2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/520d27b2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/520d27b2
Branch: refs/heads/branch-1
Commit: 520d27b267737e43718723692bfdb161139c4de6
Parents: 1972996
Author: Apekshit
Authored: Wed Jun 1 21:52:29 2016 -0700
Committer: Apekshit Sharma
Committed: Fri Jun 3 23:27:23 2016 -0700
--
.../regionserver/TestRegionServerMetrics.java | 596 +++
1 file changed, 209 insertions(+), 387 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/520d27b2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index d068217..b7b72a6 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -17,20 +17,29 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
import static org.junit.Assert.*;
@@ -38,23 +47,36 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
-@Category(MediumTests.class)
+@Category({RegionServerTests.class, MediumTests.class})
public class TestRegionServerMetrics {
- private static MetricsAssertHelper metricsHelper;
+ private static final Log LOG =
LogFactory.getLog(TestRegionServerMetrics.class);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @ClassRule
+ public static TestRule timeout = CategoryBasedTimeout.builder()
+
.withTimeout(TestRegionServerMetrics.class).withLookingForStuckThread(true).build();
static {
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
}
+ private static MetricsAssertHelper metricsHelper;
private static MiniHBaseCluster cluster;
private static HRegionServer rs;
private static Configuration conf;
private static HBaseTestingUtility TEST_UTIL;
+ private static Connection connection;
private static MetricsRegionServer metricsRegionServer;
private static MetricsRegionServerSource serverSource;
private static final int NUM_SCAN_NEXT = 30;
private static int numScanNext = 0;
+ private static byte[] cf = Bytes.toBytes("cf");
+ private static byte[] row = Bytes.toBytes("row");
+ private static byte[] qualifier = Bytes.toBytes("qual");
+ private static byte[] val = Bytes.toBytes("val");
+ private static Admin admin;
@BeforeClass
public static void startCluster() throws Exception {
@@ -64,12 +86,16 @@ public class TestRegionServerMetrics {
conf.getLong("hbase.splitlog.max.resubmit", 0);
// Make the failure test faster
conf.setInt("zookeeper.recovery.retry", 0);
+// testMobMetrics creates few hfiles and manages compaction manually.
+conf.setInt("hbase.hstore.compactionThreshold", 100);
+conf.setInt("hbase.hstore.compaction.max", 100);
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
TEST_UTIL.startMiniCluster(1, 1);
cluster = TEST_UTIL.getHBaseCluster();
-
cluster.waitForActiveAndReadyMaster();
+admin = TEST_UTIL.getHBaseAdmin();
+connection = TEST_UTIL.getConnection();
while (cluster.getLiveRegionServerThreads().size() < 1) {
Threads.sleep(100);
@@ -87,85 +1
hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called
Repository: hbase Updated Branches: refs/heads/branch-1 2ee7d2a48 -> 1972996ff HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called from within every single command and use Ruby's 'yield' magic. Instead, we can simplify it using 'command_safe' function. Since command_safe wraps all commands, we can simply time before and after we call individual command. If a command only wants to time a part of its logic, it can set instance variables start_time and end_time accordingly which is far more simpler to understand and work with than 'yield'. Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1972996f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1972996f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1972996f Branch: refs/heads/branch-1 Commit: 1972996ffcbcbd13327c277321a687e7559991ca Parents: 2ee7d2a Author: Apekshit Sharma Authored: Fri Jun 3 14:13:45 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 16:59:34 2016 -0700 -- hbase-shell/src/main/ruby/shell.rb | 7 ++--- hbase-shell/src/main/ruby/shell/commands.rb | 27 +++- .../main/ruby/shell/commands/abort_procedure.rb | 6 + .../src/main/ruby/shell/commands/add_labels.rb | 4 +-- .../src/main/ruby/shell/commands/add_peer.rb| 4 +-- .../src/main/ruby/shell/commands/alter.rb | 4 +-- .../src/main/ruby/shell/commands/alter_async.rb | 4 +-- .../main/ruby/shell/commands/alter_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/append.rb | 10 .../ruby/shell/commands/append_peer_tableCFs.rb | 4 +-- .../src/main/ruby/shell/commands/assign.rb | 4 +-- .../main/ruby/shell/commands/balance_switch.rb | 6 + .../src/main/ruby/shell/commands/balancer.rb| 14 +- .../ruby/shell/commands/balancer_enabled.rb | 6 + .../shell/commands/catalogjanitor_enabled.rb| 6 + .../ruby/shell/commands/catalogjanitor_run.rb | 4 +-- .../shell/commands/catalogjanitor_switch.rb | 6 + .../src/main/ruby/shell/commands/clear_auths.rb | 4 +-- .../main/ruby/shell/commands/clone_snapshot.rb | 4 +-- .../main/ruby/shell/commands/close_region.rb| 4 +-- .../src/main/ruby/shell/commands/compact.rb | 4 +-- .../src/main/ruby/shell/commands/compact_rs.rb | 4 +-- .../src/main/ruby/shell/commands/count.rb | 4 +-- .../src/main/ruby/shell/commands/create.rb | 7 +++-- .../ruby/shell/commands/create_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/delete.rb | 13 +- .../ruby/shell/commands/delete_all_snapshot.rb | 6 ++--- .../main/ruby/shell/commands/delete_snapshot.rb | 4 +-- .../shell/commands/delete_table_snapshots.rb| 19 +++--- .../src/main/ruby/shell/commands/deleteall.rb | 5 ++-- .../src/main/ruby/shell/commands/describe.rb| 4 +-- .../ruby/shell/commands/describe_namespace.rb | 3 --- .../src/main/ruby/shell/commands/disable.rb | 4 +-- .../main/ruby/shell/commands/disable_peer.rb| 4 +-- .../shell/commands/disable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/drop.rb| 4 +-- .../main/ruby/shell/commands/drop_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/enable.rb | 4 +-- .../src/main/ruby/shell/commands/enable_peer.rb | 4 +-- .../shell/commands/enable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/exists.rb | 4 +-- .../src/main/ruby/shell/commands/flush.rb | 4 +-- hbase-shell/src/main/ruby/shell/commands/get.rb | 4 +-- .../src/main/ruby/shell/commands/get_auths.rb | 8 +++--- .../main/ruby/shell/commands/get_peer_config.rb | 7 +++-- .../src/main/ruby/shell/commands/get_table.rb | 4 +-- .../src/main/ruby/shell/commands/grant.rb | 9 +++ .../src/main/ruby/shell/commands/incr.rb| 12 - .../src/main/ruby/shell/commands/is_disabled.rb | 8 ++ .../src/main/ruby/shell/commands/is_enabled.rb | 6 + .../src/main/ruby/shell/commands/list.rb| 3 +-- .../src/main/ruby/shell/commands/list_labels.rb | 8 +++--- .../main/ruby/shell/commands/list_namespace.rb | 3 +-- .../shell/commands/list_namespace_tables.rb | 3 +-- .../ruby/shell/commands/list_peer_configs.rb| 18 ++--- .../src/main/ruby/shell/commands/list_peers.rb | 3 +-- .../main/ruby/shell/commands/list_procedures.rb | 3 +-- .../src/main/ruby/shell/commands/list_quotas.rb | 3 +-- .../shell/commands/list_replicated_tables.rb| 4 +-- .../main/ruby/shell/commands/list_snapshots.rb | 3 +-- .../ruby/shell/commands/list_table_snapshots.rb | 3 +-- .../main/ruby/shell/commands/locate_region.rb | 4 +-- .
hbase git commit: HBASE-15845 Changes: - Remove omnipresence of formatter object since it is kind of a use-and-throw class. Commands should create an instance, use it to format the output and discard
Repository: hbase Updated Branches: refs/heads/branch-1 fa182c2ea -> 2ee7d2a48 HBASE-15845 Changes: - Remove omnipresence of formatter object since it is kind of a use-and-throw class. Commands should create an instance, use it to format the output and discard it. - Some refactoring Tested: Ran TestShell Change-Id: I12176ee12695df952ee4b0f8397e5d3be8f16a87 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2ee7d2a4 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2ee7d2a4 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2ee7d2a4 Branch: refs/heads/branch-1 Commit: 2ee7d2a4882b18001bd5fbe7e5db32d6f75153ca Parents: fa182c2 Author: Apekshit Sharma Authored: Fri Jun 3 16:34:52 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 16:35:29 2016 -0700 -- bin/hirb.rb | 6 +- hbase-shell/src/main/ruby/hbase/admin.rb| 82 hbase-shell/src/main/ruby/hbase/hbase.rb| 34 hbase-shell/src/main/ruby/hbase/quotas.rb | 3 +- .../src/main/ruby/hbase/replication_admin.rb| 3 +- hbase-shell/src/main/ruby/hbase/security.rb | 3 +- hbase-shell/src/main/ruby/hbase/table.rb| 2 +- hbase-shell/src/main/ruby/hbase/taskmonitor.rb | 3 +- .../src/main/ruby/hbase/visibility_labels.rb| 5 +- hbase-shell/src/main/ruby/shell.rb | 23 +++--- hbase-shell/src/main/ruby/shell/commands.rb | 9 ++- hbase-shell/src/test/ruby/hbase/admin_test.rb | 1 - hbase-shell/src/test/ruby/hbase/hbase_test.rb | 9 +-- .../test/ruby/hbase/replication_admin_test.rb | 1 - .../src/test/ruby/hbase/security_admin_test.rb | 1 - .../ruby/hbase/visibility_labels_admin_test.rb | 1 - .../src/test/ruby/shell/noninteractive_test.rb | 4 +- hbase-shell/src/test/ruby/shell/shell_test.rb | 4 +- hbase-shell/src/test/ruby/test_helper.rb| 8 +- 19 files changed, 86 insertions(+), 116 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2ee7d2a4/bin/hirb.rb -- diff --git a/bin/hirb.rb b/bin/hirb.rb index 94b5cdb..2726e48 100644 --- a/bin/hirb.rb +++ b/bin/hirb.rb @@ -123,15 +123,11 @@ require 'shell' # Require formatter require 'shell/formatter' -# Presume console format. -# Formatter takes an :output_stream parameter, if you don't want STDOUT. -@formatter = Shell::Formatter::Console.new - # Setup the HBase module. Create a configuration. @hbase = Hbase::Hbase.new # Setup console -@shell = Shell::Shell.new(@hbase, @formatter, interactive) +@shell = Shell::Shell.new(@hbase, interactive) @shell.debug = @shell_debug # Add commands to this namespace http://git-wip-us.apache.org/repos/asf/hbase/blob/2ee7d2a4/hbase-shell/src/main/ruby/hbase/admin.rb -- diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index f2bca0e..21e0652 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -32,10 +32,10 @@ module Hbase class Admin include HBaseConstants -def initialize(admin, formatter) - @admin = admin - @connection = @admin.getConnection() - @formatter = formatter +def initialize(connection) + @connection = connection + # Java Admin instance + @admin = @connection.getAdmin end def close @@ -276,12 +276,6 @@ module Hbase end #-- -# Parse arguments and update HTableDescriptor accordingly -def parse_htd_args(htd, arg) - htd.setNormalizationEnabled(JBoolean.valueOf(arg.delete(NORMALIZATION_ENABLED))) if arg[NORMALIZATION_ENABLED] -end - - #-- # Creates a table def create(table_name, *args) # Fail if table name is not a string @@ -359,24 +353,7 @@ module Hbase end # Done with splits; apply formerly-table_att parameters. -htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] -htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] -htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] -htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] - htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] -# DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. -# However, it has to be set before DURABILI
hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 1dfde7587 -> 73a746239
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active
cleanup scan to finish (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73a74623
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73a74623
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73a74623
Branch: refs/heads/branch-1.1
Commit: 73a7462395f7ece990bf07ea98c5772de87df441
Parents: 1dfde75
Author: Stephen Yuan Jiang
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang
Committed: Fri Jun 3 15:59:33 2016 -0700
--
.../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/73a74623/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
/**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an
active
+// scan still going on, callers might not be aware of that and do further
action thinkng
+// that no action would be from this chore. In this case, the right
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+ while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+ }
+}
+return alreadyEnabled;
}
boolean getEnabled() {
hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 b41bf10ed -> edbf2754a
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active
cleanup scan to finish (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/edbf2754
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/edbf2754
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/edbf2754
Branch: refs/heads/branch-1.2
Commit: edbf2754a410d3a4496ba1c2c8f96c7919da50a8
Parents: b41bf10
Author: Stephen Yuan Jiang
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang
Committed: Fri Jun 3 15:57:11 2016 -0700
--
.../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/edbf2754/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
/**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an
active
+// scan still going on, callers might not be aware of that and do further
action thinkng
+// that no action would be from this chore. In this case, the right
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+ while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+ }
+}
+return alreadyEnabled;
}
boolean getEnabled() {
hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 a2f9126ca -> aa636bef4
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active
cleanup scan to finish (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aa636bef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aa636bef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aa636bef
Branch: refs/heads/branch-1.3
Commit: aa636bef4ceb4254467faff8b4841275486e86bf
Parents: a2f9126
Author: Stephen Yuan Jiang
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang
Committed: Fri Jun 3 15:52:47 2016 -0700
--
.../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/aa636bef/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
/**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an
active
+// scan still going on, callers might not be aware of that and do further
action thinkng
+// that no action would be from this chore. In this case, the right
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+ while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+ }
+}
+return alreadyEnabled;
}
boolean getEnabled() {
hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1 9f343b587 -> fa182c2ea
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active
cleanup scan to finish (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa182c2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa182c2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa182c2e
Branch: refs/heads/branch-1
Commit: fa182c2ea68fcadb25beb78f4e99d55c0a507f36
Parents: 9f343b5
Author: Stephen Yuan Jiang
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang
Committed: Fri Jun 3 15:50:13 2016 -0700
--
.../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa182c2e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index d43e71a..37779c1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
/**
@@ -88,7 +89,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an
active
+// scan still going on, callers might not be aware of that and do further
action thinkng
+// that no action would be from this chore. In this case, the right
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+ while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+ }
+}
+return alreadyEnabled;
}
boolean getEnabled() {
hbase git commit: HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called
Repository: hbase Updated Branches: refs/heads/master de1b5ff77 -> 70762faa9 HBASE-15849 Simplify the way we handle runtime of commands. Functions format_simple_command and format_and_return_simple_command are used to print runtimes right now. They are called from within every single command and use Ruby's 'yield' magic. Instead, we can simplify it using 'command_safe' function. Since command_safe wraps all commands, we can simply time before and after we call individual command. If a command only wants to time a part of its logic, it can set instance variables start_time and end_time accordingly which is far more simpler to understand and work with than 'yield'. Change-Id: Ibfacf3593175af22fc4f7d80896dd2f6d7c5dde3 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70762faa Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70762faa Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70762faa Branch: refs/heads/master Commit: 70762faa98e6ad26f10fa8bd9ebc1824fdcb507c Parents: de1b5ff Author: Apekshit Sharma Authored: Fri Jun 3 14:13:45 2016 -0700 Committer: Apekshit Sharma Committed: Fri Jun 3 15:50:03 2016 -0700 -- hbase-shell/src/main/ruby/shell.rb | 7 ++--- hbase-shell/src/main/ruby/shell/commands.rb | 27 +++- .../main/ruby/shell/commands/abort_procedure.rb | 6 + .../src/main/ruby/shell/commands/add_labels.rb | 4 +-- .../src/main/ruby/shell/commands/add_peer.rb| 4 +-- .../src/main/ruby/shell/commands/alter.rb | 4 +-- .../src/main/ruby/shell/commands/alter_async.rb | 4 +-- .../main/ruby/shell/commands/alter_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/append.rb | 10 .../ruby/shell/commands/append_peer_tableCFs.rb | 4 +-- .../src/main/ruby/shell/commands/assign.rb | 4 +-- .../main/ruby/shell/commands/balance_switch.rb | 6 + .../src/main/ruby/shell/commands/balancer.rb| 14 +- .../ruby/shell/commands/balancer_enabled.rb | 6 + .../shell/commands/catalogjanitor_enabled.rb| 6 + .../ruby/shell/commands/catalogjanitor_run.rb | 4 +-- .../shell/commands/catalogjanitor_switch.rb | 6 + .../src/main/ruby/shell/commands/clear_auths.rb | 4 +-- .../main/ruby/shell/commands/clone_snapshot.rb | 4 +-- .../main/ruby/shell/commands/close_region.rb| 4 +-- .../src/main/ruby/shell/commands/compact.rb | 4 +-- .../src/main/ruby/shell/commands/compact_rs.rb | 4 +-- .../src/main/ruby/shell/commands/count.rb | 4 +-- .../src/main/ruby/shell/commands/create.rb | 7 +++-- .../ruby/shell/commands/create_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/delete.rb | 13 +- .../ruby/shell/commands/delete_all_snapshot.rb | 6 ++--- .../main/ruby/shell/commands/delete_snapshot.rb | 4 +-- .../shell/commands/delete_table_snapshots.rb| 19 +++--- .../src/main/ruby/shell/commands/deleteall.rb | 5 ++-- .../src/main/ruby/shell/commands/describe.rb| 4 +-- .../ruby/shell/commands/describe_namespace.rb | 3 --- .../src/main/ruby/shell/commands/disable.rb | 4 +-- .../main/ruby/shell/commands/disable_peer.rb| 4 +-- .../shell/commands/disable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/drop.rb| 4 +-- .../main/ruby/shell/commands/drop_namespace.rb | 4 +-- .../src/main/ruby/shell/commands/enable.rb | 4 +-- .../src/main/ruby/shell/commands/enable_peer.rb | 4 +-- .../shell/commands/enable_table_replication.rb | 6 ++--- .../src/main/ruby/shell/commands/exists.rb | 4 +-- .../src/main/ruby/shell/commands/flush.rb | 4 +-- hbase-shell/src/main/ruby/shell/commands/get.rb | 4 +-- .../src/main/ruby/shell/commands/get_auths.rb | 8 +++--- .../main/ruby/shell/commands/get_peer_config.rb | 7 +++-- .../src/main/ruby/shell/commands/get_table.rb | 4 +-- .../src/main/ruby/shell/commands/grant.rb | 9 +++ .../src/main/ruby/shell/commands/incr.rb| 12 - .../src/main/ruby/shell/commands/is_disabled.rb | 8 ++ .../src/main/ruby/shell/commands/is_enabled.rb | 6 + .../src/main/ruby/shell/commands/list.rb| 3 +-- .../src/main/ruby/shell/commands/list_labels.rb | 8 +++--- .../main/ruby/shell/commands/list_namespace.rb | 3 +-- .../shell/commands/list_namespace_tables.rb | 3 +-- .../ruby/shell/commands/list_peer_configs.rb| 18 ++--- .../src/main/ruby/shell/commands/list_peers.rb | 3 +-- .../main/ruby/shell/commands/list_procedures.rb | 3 +-- .../src/main/ruby/shell/commands/list_quotas.rb | 3 +-- .../shell/commands/list_replicated_tables.rb| 4 +-- .../main/ruby/shell/commands/list_snapshots.rb | 3 +-- .../ruby/shell/commands/list_table_snapshots.rb | 3 +-- .../main/ruby/shell/commands/locate_region.rb | 4 +-- .../m
hbase git commit: HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active cleanup scan to finish (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/master 21e98271c -> de1b5ff77
HBASE-15955 Disable action in CatalogJanitor#setEnabled should wait for active
cleanup scan to finish (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de1b5ff7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de1b5ff7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de1b5ff7
Branch: refs/heads/master
Commit: de1b5ff7761b899691085e147c885e15897a562c
Parents: 21e9827
Author: Stephen Yuan Jiang
Authored: Fri Jun 3 15:48:48 2016 -0700
Committer: Stephen Yuan Jiang
Committed: Fri Jun 3 15:49:23 2016 -0700
--
.../org/apache/hadoop/hbase/master/CatalogJanitor.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/de1b5ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index b9abc65..c93b307 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Triple;
/**
@@ -86,7 +87,17 @@ public class CatalogJanitor extends ScheduledChore {
* @param enabled
*/
public boolean setEnabled(final boolean enabled) {
-return this.enabled.getAndSet(enabled);
+boolean alreadyEnabled = this.enabled.getAndSet(enabled);
+// If disabling is requested on an already enabled chore, we could have an
active
+// scan still going on, callers might not be aware of that and do further
action thinkng
+// that no action would be from this chore. In this case, the right
action is to wait for
+// the active scan to complete before exiting this function.
+if (!enabled && alreadyEnabled) {
+ while (alreadyRunning.get()) {
+Threads.sleepWithoutInterrupt(100);
+ }
+}
+return alreadyEnabled;
}
boolean getEnabled() {
hbase git commit: HBASE-15881 Allow BZIP2 compression.
Repository: hbase
Updated Branches:
refs/heads/0.98 67dc17240 -> 804f8e7fd
HBASE-15881 Allow BZIP2 compression.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/804f8e7f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/804f8e7f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/804f8e7f
Branch: refs/heads/0.98
Commit: 804f8e7fd41e7872e0080cc3dd884c70c1fe29e9
Parents: 67dc172
Author: Lars Hofhansl
Authored: Fri Jun 3 15:37:47 2016 -0700
Committer: Lars Hofhansl
Committed: Fri Jun 3 15:37:47 2016 -0700
--
.../hadoop/hbase/io/compress/Compression.java | 29 +++-
.../hadoop/hbase/util/TestCompressionTest.java | 2 ++
2 files changed, 30 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/804f8e7f/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index be7e5f6..edb0f48 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
throw new RuntimeException(e);
}
}
- };
+},
+BZIP2("bzip2") {
+ // Use base type to avoid compile-time dependencies.
+ private volatile transient CompressionCodec bzipCodec;
+ private transient Object lock = new Object();
+
+ @Override
+ CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+ synchronized (lock) {
+if (bzipCodec == null) {
+ bzipCodec = buildCodec(conf);
+}
+ }
+}
+return bzipCodec;
+ }
+
+ private CompressionCodec buildCodec(Configuration conf) {
+try {
+ Class externalCodec =
+
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+ return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
conf);
+} catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+}
+ }
+};
private final Configuration conf;
private final String compressName;
http://git-wip-us.apache.org/repos/asf/hbase/blob/804f8e7f/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index d5f3442..6ce8318 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
nativeCodecTest("SNAPPY", "snappy",
"org.apache.hadoop.io.compress.SnappyCodec");
+ nativeCodecTest("BZIP2", "bzip2",
"org.apache.hadoop.io.compress.BZip2Codec");
} else {
// Hadoop nativelib is not available
LOG.debug("Native code not loaded");
assertFalse(CompressionTest.testCompression("LZO"));
assertFalse(CompressionTest.testCompression("LZ4"));
assertFalse(CompressionTest.testCompression("SNAPPY"));
+ assertFalse(CompressionTest.testCompression("BZIP2"));
}
}
[2/3] hbase git commit: HBASE-15881 Allow BZIP2 compression.
HBASE-15881 Allow BZIP2 compression.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f343b58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f343b58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f343b58
Branch: refs/heads/branch-1
Commit: 9f343b5871033e1be166e6ab605189c89bbcf7d5
Parents: 7bcdd44
Author: Lars Hofhansl
Authored: Thu Jun 2 12:19:02 2016 -0700
Committer: Lars Hofhansl
Committed: Fri Jun 3 15:33:05 2016 -0700
--
.../hadoop/hbase/io/compress/Compression.java | 31 ++--
.../hadoop/hbase/util/TestCompressionTest.java | 2 ++
2 files changed, 31 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f343b58/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index c6ebff5..099b445 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
throw new RuntimeException(e);
}
}
- };
+},
+BZIP2("bzip2") {
+ // Use base type to avoid compile-time dependencies.
+ private volatile transient CompressionCodec bzipCodec;
+ private transient Object lock = new Object();
+
+ @Override
+ CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+ synchronized (lock) {
+if (bzipCodec == null) {
+ bzipCodec = buildCodec(conf);
+}
+ }
+}
+return bzipCodec;
+ }
+
+ private CompressionCodec buildCodec(Configuration conf) {
+try {
+ Class externalCodec =
+
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+ return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
conf);
+} catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+}
+ }
+};
private final transient Configuration conf; // FindBugs: SE_BAD_FIELD so
just made it transient
private final String compressName;
@@ -431,4 +458,4 @@ public final class Compression {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f343b58/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index 682b066..b8e79ed 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
nativeCodecTest("SNAPPY", "snappy",
"org.apache.hadoop.io.compress.SnappyCodec");
+ nativeCodecTest("BZIP2", "bzip2",
"org.apache.hadoop.io.compress.BZip2Codec");
} else {
// Hadoop nativelib is not available
LOG.debug("Native code not loaded");
assertFalse(CompressionTest.testCompression("LZO"));
assertFalse(CompressionTest.testCompression("LZ4"));
assertFalse(CompressionTest.testCompression("SNAPPY"));
+ assertFalse(CompressionTest.testCompression("BZIP2"));
}
}
[3/3] hbase git commit: HBASE-15881 Allow BZIP2 compression.
HBASE-15881 Allow BZIP2 compression.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2f9126c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2f9126c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2f9126c
Branch: refs/heads/branch-1.3
Commit: a2f9126cad196c6e305b15615b8165fda8a47914
Parents: b0e1fda
Author: Lars Hofhansl
Authored: Thu Jun 2 12:19:02 2016 -0700
Committer: Lars Hofhansl
Committed: Fri Jun 3 15:33:31 2016 -0700
--
.../hadoop/hbase/io/compress/Compression.java | 31 ++--
.../hadoop/hbase/util/TestCompressionTest.java | 2 ++
2 files changed, 31 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2f9126c/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
index c6ebff5..099b445 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
@@ -235,7 +235,34 @@ public final class Compression {
throw new RuntimeException(e);
}
}
- };
+},
+BZIP2("bzip2") {
+ // Use base type to avoid compile-time dependencies.
+ private volatile transient CompressionCodec bzipCodec;
+ private transient Object lock = new Object();
+
+ @Override
+ CompressionCodec getCodec(Configuration conf) {
+if (bzipCodec == null) {
+ synchronized (lock) {
+if (bzipCodec == null) {
+ bzipCodec = buildCodec(conf);
+}
+ }
+}
+return bzipCodec;
+ }
+
+ private CompressionCodec buildCodec(Configuration conf) {
+try {
+ Class externalCodec =
+
getClassLoaderForCodec().loadClass("org.apache.hadoop.io.compress.BZip2Codec");
+ return (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
conf);
+} catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+}
+ }
+};
private final transient Configuration conf; // FindBugs: SE_BAD_FIELD so
just made it transient
private final String compressName;
@@ -431,4 +458,4 @@ public final class Compression {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2f9126c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
index 682b066..b8e79ed 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
@@ -74,12 +74,14 @@ public class TestCompressionTest {
nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
nativeCodecTest("SNAPPY", "snappy",
"org.apache.hadoop.io.compress.SnappyCodec");
+ nativeCodecTest("BZIP2", "bzip2",
"org.apache.hadoop.io.compress.BZip2Codec");
} else {
// Hadoop nativelib is not available
LOG.debug("Native code not loaded");
assertFalse(CompressionTest.testCompression("LZO"));
assertFalse(CompressionTest.testCompression("LZ4"));
assertFalse(CompressionTest.testCompression("SNAPPY"));
+ assertFalse(CompressionTest.testCompression("BZIP2"));
}
}
[1/3] hbase git commit: HBASE-14307 - Incorrect use of positional read api in HFileBlock (Chris Nauroth)
Repository: hbase
Updated Branches:
refs/heads/branch-1 7bcdd44be -> 9f343b587
refs/heads/branch-1.3 b0e1fdae3 -> a2f9126ca
refs/heads/branch-2 [created] 75bb6d96b
HBASE-14307 - Incorrect use of positional read api in HFileBlock (Chris
Nauroth)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75bb6d96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75bb6d96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75bb6d96
Branch: refs/heads/branch-2
Commit: 75bb6d96bcf974dd04da3543638dd1bf7d9a6156
Parents: 3703ad4
Author: ramkrishna
Authored: Fri Sep 11 15:56:09 2015 +0530
Committer: ramkrishna
Committed: Fri Sep 11 16:01:56 2015 +0530
--
.../hadoop/hbase/io/hfile/HFileBlock.java | 50 ++-
.../io/hfile/TestHFileBlockPositionalRead.java | 148 +++
2 files changed, 190 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/75bb6d96/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 1415a00..2da91b6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.io.IOUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
@@ -695,6 +696,45 @@ public class HFileBlock implements Cacheable {
}
/**
+ * Read from an input stream. Analogous to
+ * {@link IOUtils#readFully(InputStream, byte[], int, int)}, but uses
+ * positional read and specifies a number of "extra" bytes that would be
+ * desirable but not absolutely necessary to read.
+ *
+ * @param in the input stream to read from
+ * @param position the position within the stream from which to start reading
+ * @param buf the buffer to read into
+ * @param bufOffset the destination offset in the buffer
+ * @param necessaryLen the number of bytes that are absolutely necessary to
+ * read
+ * @param extraLen the number of extra bytes that would be nice to read
+ * @return true if and only if extraLen is > 0 and reading those extra bytes
+ * was successful
+ * @throws IOException if failed to read the necessary bytes
+ */
+ @VisibleForTesting
+ static boolean positionalReadWithExtra(FSDataInputStream in,
+ long position, byte[] buf, int bufOffset, int necessaryLen, int extraLen)
+ throws IOException {
+int bytesRemaining = necessaryLen + extraLen;
+int bytesRead = 0;
+while (bytesRead < necessaryLen) {
+ int ret = in.read(position, buf, bufOffset, bytesRemaining);
+ if (ret < 0) {
+throw new IOException("Premature EOF from inputStream (positional read
"
++ "returned " + ret + ", was trying to read " + necessaryLen
++ " necessary bytes and " + extraLen + " extra bytes, "
++ "successfully read " + bytesRead);
+ }
+ position += ret;
+ bufOffset += ret;
+ bytesRemaining -= ret;
+ bytesRead += ret;
+}
+return bytesRead != necessaryLen && bytesRemaining <= 0;
+ }
+
+ /**
* @return the on-disk size of the next block (including the header size)
* that was read by peeking into the next block's header
*/
@@ -1377,14 +1417,8 @@ public class HFileBlock implements Cacheable {
} else {
// Positional read. Better for random reads; or when the streamLock is
already locked.
int extraSize = peekIntoNextBlock ? hdrSize : 0;
-int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
-if (ret < size) {
- throw new IOException("Positional read of " + size + " bytes " +
- "failed at offset " + fileOffset + " (returned " + ret + ")");
-}
-
-if (ret == size || ret < size + extraSize) {
- // Could not read the next block's header, or did not try.
+if (!positionalReadWithExtra(istream, fileOffset, dest, destOffset,
+size, extraSize)) {
return -1;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/75bb6d96/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockPositionalRead.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockPositionalRead.java
b/hbase-server/src/test/java/org
hbase git commit: HBASE-15883 Adding WAL files and tracking offsets in HBase.
Repository: hbase
Updated Branches:
refs/heads/master 9a53d8b38 -> 21e98271c
HBASE-15883 Adding WAL files and tracking offsets in HBase.
Implemented ReplicationQueuesHBaseImpl that tracks WAL offsets and replication
queues in an HBase table.
Only wrote the basic tracking methods, have not implemented claimQueue() or
HFileRef methods yet.
Wrote a basic unit test for ReplicationQueueHBaseImpl that tests the
implemented functions on a single Region Server
Signed-off-by: Elliott Clark
Signed-off-by: Elliott Clark
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21e98271
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21e98271
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21e98271
Branch: refs/heads/master
Commit: 21e98271c32f0d44106515a72b2c92d518c03668
Parents: 9a53d8b
Author: Joseph Hwang
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark
Committed: Fri Jun 3 15:23:10 2016 -0700
--
.../hbase/replication/ReplicationFactory.java | 11 +-
.../hbase/replication/ReplicationQueues.java| 8 +-
.../replication/ReplicationQueuesArguments.java | 66 +++
.../replication/ReplicationQueuesHBaseImpl.java | 491 +++
.../replication/ReplicationQueuesZKImpl.java| 13 +-
.../replication/regionserver/Replication.java | 12 +-
.../regionserver/ReplicationSourceManager.java | 5 +-
.../replication/TestReplicationAdmin.java | 3 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 3 +-
.../cleaner/TestReplicationHFileCleaner.java| 4 +-
.../replication/TestReplicationStateBasic.java | 2 +-
.../TestReplicationStateHBaseImpl.java | 243 +
.../replication/TestReplicationStateZKImpl.java | 13 +-
.../TestReplicationSourceManager.java | 36 +-
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 4 +-
15 files changed, 871 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 91e77ca..e264a4d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.replication;
+import org.apache.commons.lang.reflect.ConstructorUtils;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
@@ -30,9 +31,11 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationQueues getReplicationQueues(final ZooKeeperWatcher
zk,
- Configuration conf, Abortable abortable) {
-return new ReplicationQueuesZKImpl(zk, conf, abortable);
+ public static ReplicationQueues
getReplicationQueues(ReplicationQueuesArguments args)
+ throws Exception {
+Class classToBuild = args.getConf().getClass("hbase.region.replica." +
+"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class);
+return (ReplicationQueues)
ConstructorUtils.invokeConstructor(classToBuild, args);
}
public static ReplicationQueuesClient getReplicationQueuesClient(final
ZooKeeperWatcher zk,
@@ -44,7 +47,7 @@ public class ReplicationFactory {
Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
}
-
+
public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher
zk, Configuration conf,
final ReplicationQueuesClient queuesClient, Abortable abortable) {
return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0d47a88..db6da91 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -83,13 +83,13 @@ public interface ReplicationQueues {
/**
* Get a list of all WALs in the given queue.
* @param queueId a String that identifie
hbase git commit: HBASE-15929 There are two tests named TestRegionServerMetrics. This causes slight issues when parsing console output for hanging tests and flaky tests analysis. Moving .../hbase/Test
Repository: hbase
Updated Branches:
refs/heads/master b557f0bec -> 9a53d8b38
HBASE-15929 There are two tests named TestRegionServerMetrics. This causes
slight issues when parsing console output for hanging tests and flaky tests
analysis. Moving .../hbase/TestRegionServerMetrics.java to
.../hbase/regionserver/TestRegionServerReadRequestMetrics.java. (Apekshit)
Change-Id: I379c15fe2c2c01bed53bddf7619d5f2a07c5640e
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a53d8b3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a53d8b3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a53d8b3
Branch: refs/heads/master
Commit: 9a53d8b3850f0521c3a4ad194a3b84d7f4145007
Parents: b557f0b
Author: Apekshit
Authored: Thu Jun 2 20:06:20 2016 -0700
Committer: Apekshit Sharma
Committed: Fri Jun 3 15:13:49 2016 -0700
--
.../hadoop/hbase/TestRegionServerMetrics.java | 379 --
.../TestRegionServerReadRequestMetrics.java | 387 +++
2 files changed, 387 insertions(+), 379 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9a53d8b3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
deleted file mode 100644
index 76e5842..000
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionServerMetrics.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-@Category(MediumTests.class)
-public class TestRegionServerMetrics {
- private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
- private static final TableName TABLE_NAME = TableName.valueOf("test");
- private static final byte[] CF1 = "c1".getBytes();
- private static final byte[] CF2 = "c2".getBytes();
-
- private static final byte[] ROW1 = "a".getBytes();
- private static final byte[] ROW2 = "b".getBytes();
- private static final byte[] ROW3 = "c".getBytes();
- private static final byte[] COL1 = "q1".getBytes();
- private static final byte[] COL2 = "q2".getBytes();
- private static final byte[] COL3 = "q3".getBytes();
- private static final byte[] VAL1 = "v1".getBytes();
- private static final byte[] VAL2 = "v2".getBytes();
- private static final byte[] VAL3 = Bytes.toBytes(0L);
-
- private static final int MAX_TRY = 20;
- private static final int SLEEP_MS = 100;
- private static final int TTL = 1;
-
- private static Admin admin;
- private static Collection serverNames;
- private static Table table;
- private static List tableRegions;
-
- private static Map requestsMap = new HashMap<>
hbase git commit: HBASE-15949 Cleanup TestRegionServerMetrics. @Before and @After to setup/teardown tables using @Rule to set table name based on testname. Refactor out copy-pasted code fragments to s
Repository: hbase
Updated Branches:
refs/heads/master bdb46f01b -> b557f0bec
HBASE-15949 Cleanup TestRegionServerMetrics.
@Before and @After to setup/teardown tables using @Rule to set table name based
on testname.
Refactor out copy-pasted code fragments to single function.
(Apekshit)
Change-Id: Ic22e5027cc3952bab5ec30070ed20e98017db65a
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b557f0be
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b557f0be
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b557f0be
Branch: refs/heads/master
Commit: b557f0bec62a48753e5d01d7a47f3c9e5a6b3ee8
Parents: bdb46f0
Author: Apekshit
Authored: Wed Jun 1 21:52:29 2016 -0700
Committer: Apekshit Sharma
Committed: Fri Jun 3 14:27:24 2016 -0700
--
.../regionserver/TestRegionServerMetrics.java | 682 +++
1 file changed, 256 insertions(+), 426 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b557f0be/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 18796bd..06db468 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
@@ -27,11 +29,17 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
import static org.junit.Assert.*;
@@ -39,23 +47,35 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
@Category({RegionServerTests.class, MediumTests.class})
public class TestRegionServerMetrics {
- private static MetricsAssertHelper metricsHelper;
+ private static final Log LOG =
LogFactory.getLog(TestRegionServerMetrics.class);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @ClassRule
+ public static TestRule timeout =
CategoryBasedTimeout.forClass(TestRegionServerMetrics.class);
static {
Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
}
+ private static MetricsAssertHelper metricsHelper;
private static MiniHBaseCluster cluster;
private static HRegionServer rs;
private static Configuration conf;
private static HBaseTestingUtility TEST_UTIL;
+ private static Connection connection;
private static MetricsRegionServer metricsRegionServer;
private static MetricsRegionServerSource serverSource;
private static final int NUM_SCAN_NEXT = 30;
private static int numScanNext = 0;
+ private static byte[] cf = Bytes.toBytes("cf");
+ private static byte[] row = Bytes.toBytes("row");
+ private static byte[] qualifier = Bytes.toBytes("qual");
+ private static byte[] val = Bytes.toBytes("val");
+ private static Admin admin;
@BeforeClass
public static void startCluster() throws Exception {
@@ -65,12 +85,16 @@ public class TestRegionServerMetrics {
conf.getLong("hbase.splitlog.max.resubmit", 0);
// Make the failure test faster
conf.setInt("zookeeper.recovery.retry", 0);
+// testMobMetrics creates few hfiles and manages compaction manually.
+conf.setInt("hbase.hstore.compactionThreshold", 100);
+conf.setInt("hbase.hstore.compaction.max", 100);
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
TEST_UTIL.startMiniCluster(1, 1);
cluster = TEST_UTIL.getHBaseCluster();
-
cluster.waitForActiveAndReadyMaster();
+admin = TEST_UTIL.getHBaseAdmin();
+connection = TEST_UTIL.getConnection();
while (cluster.getLiveRegionServerThreads().size() < 1) {
Threads.sleep(100);
@@ -88,551 +112,370 @@ public class TestRegionServerMetrics {
}
}
- @Test(timeout = 30)
+ TableName tableName;
+ Table table;
+
+ @Before
+ public void beforeTestMethod() throws Exception {
+metricsRegionServer.getRegionServerWrapper().forceRecompu
hbase git commit: HBASE-15845 Changes: - Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files right now which is confusing. - Remove omnipresence of formatter object since it i
Repository: hbase
Updated Branches:
refs/heads/master f0c159b5f -> bdb46f01b
HBASE-15845 Changes:
- Renaming hbase.rb to hbase_constants.rb because there are two hbase.rb files
right now which is confusing.
- Remove omnipresence of formatter object since it is kind of a use-and-throw
class. Commands should create
an instance, use it to format the output and discard it.
- Some refactoring
Change-Id: If9ea9873904e0a39d199a6aa10e23864b86a2f09
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdb46f01
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdb46f01
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdb46f01
Branch: refs/heads/master
Commit: bdb46f01b9645a0aa4e9827a7e627de76cd7983b
Parents: f0c159b
Author: Apekshit
Authored: Mon Dec 28 14:50:50 2015 -0800
Committer: Apekshit Sharma
Committed: Fri Jun 3 13:37:38 2016 -0700
--
bin/hirb.rb | 8 +-
hbase-shell/src/main/ruby/hbase.rb | 109 ---
hbase-shell/src/main/ruby/hbase/admin.rb| 82 ++
hbase-shell/src/main/ruby/hbase/hbase.rb| 36 +++---
hbase-shell/src/main/ruby/hbase/quotas.rb | 3 +-
.../src/main/ruby/hbase/replication_admin.rb| 3 +-
.../src/main/ruby/hbase/rsgroup_admin.rb| 3 +-
hbase-shell/src/main/ruby/hbase/security.rb | 3 +-
hbase-shell/src/main/ruby/hbase/table.rb| 2 +-
hbase-shell/src/main/ruby/hbase/taskmonitor.rb | 3 +-
.../src/main/ruby/hbase/visibility_labels.rb| 5 +-
hbase-shell/src/main/ruby/hbase_constants.rb| 109 +++
hbase-shell/src/main/ruby/shell.rb | 25 ++---
hbase-shell/src/main/ruby/shell/commands.rb | 9 +-
hbase-shell/src/test/ruby/hbase/admin_test.rb | 3 +-
hbase-shell/src/test/ruby/hbase/hbase_test.rb | 11 +-
.../test/ruby/hbase/replication_admin_test.rb | 3 +-
.../src/test/ruby/hbase/security_admin_test.rb | 3 +-
hbase-shell/src/test/ruby/hbase/table_test.rb | 2 +-
.../src/test/ruby/hbase/taskmonitor_test.rb | 2 +-
.../ruby/hbase/visibility_labels_admin_test.rb | 3 +-
.../src/test/ruby/shell/commands_test.rb| 2 +-
.../src/test/ruby/shell/noninteractive_test.rb | 6 +-
.../src/test/ruby/shell/rsgroup_shell_test.rb | 18 ++-
hbase-shell/src/test/ruby/shell/shell_test.rb | 6 +-
hbase-shell/src/test/ruby/test_helper.rb| 10 +-
26 files changed, 218 insertions(+), 251 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/bin/hirb.rb
--
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 94b5cdb..d0295d6 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -115,7 +115,7 @@
org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level)
org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
# Require HBase now after setting log levels
-require 'hbase'
+require 'hbase_constants'
# Load hbase shell
require 'shell'
@@ -123,15 +123,11 @@ require 'shell'
# Require formatter
require 'shell/formatter'
-# Presume console format.
-# Formatter takes an :output_stream parameter, if you don't want STDOUT.
-@formatter = Shell::Formatter::Console.new
-
# Setup the HBase module. Create a configuration.
@hbase = Hbase::Hbase.new
# Setup console
-@shell = Shell::Shell.new(@hbase, @formatter, interactive)
+@shell = Shell::Shell.new(@hbase, interactive)
@shell.debug = @shell_debug
# Add commands to this namespace
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb46f01/hbase-shell/src/main/ruby/hbase.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase.rb
b/hbase-shell/src/main/ruby/hbase.rb
deleted file mode 100644
index bc6f37c..000
--- a/hbase-shell/src/main/ruby/hbase.rb
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# HBase ruby classes.
-# Has wrapper classes for org.
[2/2] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()
HBASE-15927 Remove HMaster.assignRegion()
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7bcdd44b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7bcdd44b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7bcdd44b
Branch: refs/heads/branch-1
Commit: 7bcdd44be6951aae3f3e041b682b7d06c06f1721
Parents: 950a09b
Author: Matteo Bertozzi
Authored: Fri Jun 3 12:41:51 2016 -0700
Committer: Matteo Bertozzi
Committed: Fri Jun 3 12:41:51 2016 -0700
--
.../org/apache/hadoop/hbase/master/HMaster.java | 4 --
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/HBaseTestingUtility.java | 15 ++
.../hbase/client/TestMetaWithReplicas.java | 2 +-
.../master/TestAssignmentManagerOnCluster.java | 50
.../hadoop/hbase/master/TestMasterFailover.java | 36 +++---
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 35 --
7 files changed, 67 insertions(+), 79 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index beb1bca..4b7e181 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2387,10 +2387,6 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
return this.initializationBeforeMetaAssignment;
}
- public void assignRegion(HRegionInfo hri) {
-assignmentManager.assign(hri, true);
- }
-
/**
* Compute the average load across all region servers.
* Currently, this uses a very naive computation - just uses the number of
http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 90471cf..152379d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1389,7 +1389,7 @@ public class MasterRpcServices extends RSRpcServices
if (master.assignmentManager.getRegionStates().isRegionOffline(hri)) {
LOG.debug("Region " + hri.getRegionNameAsString()
+ " is not online on any region server, reassigning it.");
-master.assignRegion(hri);
+master.assignmentManager.assign(hri, true);
}
if (master.cpHost != null) {
master.cpHost.postUnassign(hri, force);
@@ -1572,7 +1572,7 @@ public class MasterRpcServices extends RSRpcServices
}
}
- /**
+ /**
* Returns the security capabilities in effect on the cluster
*/
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 2870593..4547e0f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3388,6 +3388,21 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
}
/**
+ * Uses directly the assignment manager to assign the region.
+ * and waits until the specified region has completed assignment.
+ * @param tableName the table name
+ * @throws IOException
+ * @throw InterruptedException
+ * @return true if the region is assigned false otherwise.
+ */
+ public boolean assignRegion(final HRegionInfo regionInfo)
+ throws IOException, InterruptedException {
+final AssignmentManager am =
getHBaseCluster().getMaster().getAssignmentManager();
+am.assign(regionInfo, true);
+return am.waitForAssignment(regionInfo);
+ }
+
+ /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,
http://git-wip-us.apache.org/repos/asf/hbase/blob/7bcdd44b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
---
[1/2] hbase git commit: HBASE-15927 Remove HMaster.assignRegion()
Repository: hbase
Updated Branches:
refs/heads/branch-1 950a09b03 -> 7bcdd44be
refs/heads/master 0cbce0762 -> f0c159b5f
HBASE-15927 Remove HMaster.assignRegion()
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0c159b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0c159b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0c159b5
Branch: refs/heads/master
Commit: f0c159b5fed732cdc08bd2fd924ae81677d66a30
Parents: 0cbce07
Author: Matteo Bertozzi
Authored: Fri Jun 3 12:36:25 2016 -0700
Committer: Matteo Bertozzi
Committed: Fri Jun 3 12:36:25 2016 -0700
--
.../org/apache/hadoop/hbase/master/HMaster.java | 4 --
.../hadoop/hbase/HBaseTestingUtility.java | 15 +
.../hbase/client/TestMetaWithReplicas.java | 2 +-
.../master/TestAssignmentManagerOnCluster.java | 61
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 20 +++
.../hadoop/hbase/util/TestHBaseFsckTwoRS.java | 23 +++-
6 files changed, 58 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f8d0003..d368ffb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2378,10 +2378,6 @@ public class HMaster extends HRegionServer implements
MasterServices {
return this.initializationBeforeMetaAssignment;
}
- public void assignRegion(HRegionInfo hri) {
-assignmentManager.assign(hri);
- }
-
/**
* Compute the average load across all region servers.
* Currently, this uses a very naive computation - just uses the number of
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 7b35815..bfa14cb 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -3200,6 +3200,21 @@ public class HBaseTestingUtility extends
HBaseCommonTestingUtility {
}
/**
+ * Uses directly the assignment manager to assign the region.
+ * and waits until the specified region has completed assignment.
+ * @param tableName the table name
+ * @throws IOException
+ * @throw InterruptedException
+ * @return true if the region is assigned false otherwise.
+ */
+ public boolean assignRegion(final HRegionInfo regionInfo)
+ throws IOException, InterruptedException {
+final AssignmentManager am =
getHBaseCluster().getMaster().getAssignmentManager();
+am.assign(regionInfo);
+return am.waitForAssignment(regionInfo);
+ }
+
+ /**
* Wait until all regions for a table in hbase:meta have a non-empty
* info:server, up to a configuable timeout value (default is 60 seconds)
* This means all regions have been deployed,
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index 5e302d2..8e87ceb 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -448,7 +448,7 @@ public class TestMetaWithReplicas {
// create in-memory state otherwise master won't assign
TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
.getRegionStates().createRegionState(h);
-TEST_UTIL.getMiniHBaseCluster().getMaster().assignRegion(h);
+TEST_UTIL.assignRegion(h);
HBaseFsckRepair.waitUntilAssigned(TEST_UTIL.getHBaseAdmin(), h);
// check that problem exists
HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0c159b5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
--
diff --git
a/hbase-serv
hbase git commit: HBASE-15953 HBase backup set list command does not list any existing sets (Vladimir)
Repository: hbase
Updated Branches:
refs/heads/HBASE-7912 c51700180 -> 964c0841d
HBASE-15953 HBase backup set list command does not list any existing sets
(Vladimir)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/964c0841
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/964c0841
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/964c0841
Branch: refs/heads/HBASE-7912
Commit: 964c0841db67464fc5b96807dfd321b44de8b5d7
Parents: c517001
Author: tedyu
Authored: Fri Jun 3 09:31:13 2016 -0700
Committer: tedyu
Committed: Fri Jun 3 09:31:13 2016 -0700
--
.../org/apache/hadoop/hbase/backup/impl/BackupCommands.java | 5 -
.../java/org/apache/hadoop/hbase/backup/TestBackupBase.java | 9 -
2 files changed, 4 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/964c0841/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 1fce171..9a8697a 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -466,7 +466,10 @@ public final class BackupCommands {
Configuration conf = getConf() != null? getConf():
HBaseConfiguration.create();
try(final Connection conn = ConnectionFactory.createConnection(conf);
final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
-admin.listBackupSets();
+List list = admin.listBackupSets();
+for(BackupSet bs: list){
+ System.out.println(bs);
+}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/964c0841/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
index 9683e16..31cf65a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
@@ -249,15 +249,6 @@ public class TestBackupBase {
}
}
-// protected BackupClient getBackupClient(){
-//return BackupRestoreClientFactory.getBackupClient(conf1);
-// }
-
-// protected RestoreClient getRestoreClient()
-// {
-//return BackupRestoreClientFactory.getRestoreClient(conf1);
-// }
-
protected BackupAdmin getBackupAdmin() throws IOException {
return TEST_UTIL.getAdmin().getBackupAdmin();
}
hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 5824f2236 -> b0e1fdae3
HBASE-15888 Extend HBASE-12769 for bulk load data replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0e1fdae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0e1fdae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0e1fdae
Branch: refs/heads/branch-1.3
Commit: b0e1fdae346b64af4188cf5df29488617753416f
Parents: 5824f22
Author: Ashish Singhi
Authored: Fri Jun 3 18:48:47 2016 +0530
Committer: Ashish Singhi
Committed: Fri Jun 3 18:48:47 2016 +0530
--
.../replication/ReplicationPeersZKImpl.java | 6 ++
.../hbase/util/hbck/ReplicationChecker.java | 59 ++--
2 files changed, 61 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e1fdae/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index ad634fa..b0d6e83 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -601,6 +601,12 @@ public class ReplicationPeersZKImpl extends
ReplicationStateZKBase implements Re
}
}
}
+ // Check for hfile-refs queue
+ if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+ && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+ }
} catch (KeeperException e) {
throw new ReplicationException("Could not check queues deleted with id="
+ peerId, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e1fdae/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index bf44a50..64212c9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
@InterfaceAudience.Private
public class ReplicationChecker {
private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+ private final ZooKeeperWatcher zkw;
private ErrorReporter errorReporter;
private ReplicationQueuesClient queuesClient;
private ReplicationPeers replicationPeers;
private ReplicationQueueDeletor queueDeletor;
// replicator with its queueIds for removed peers
private Map> undeletedQueueIds = new HashMap>();
-
+ // Set of un deleted hfile refs queue Ids
+ private Set undeletedHFileRefsQueueIds = new HashSet<>();
+ private final String hfileRefsZNode;
+
public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw,
HConnection connection,
ErrorReporter errorReporter) throws IOException {
try {
+ this.zkw = zkw;
this.errorReporter = errorReporter;
this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw,
conf, connection);
this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
} catch (ReplicationException e) {
throw new IOException("failed to construct ReplicationChecker", e);
}
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication",
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode,
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
}
public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
} catch (KeeperException ke) {
throw new IOException(ke);
}
+
+checkUnDeletedHFileRefsQueues(peerIds);
+ }
+
+ private void checkUnDeletedHFileRefsQueues(Set peerIds) throws
IOException {
+try {
+ if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+return;
+ }
+ List listOfPeers =
this.queuesClient.getAllPeersFro
hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication
Repository: hbase
Updated Branches:
refs/heads/branch-1 a8c8bfd5e -> 950a09b03
HBASE-15888 Extend HBASE-12769 for bulk load data replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/950a09b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/950a09b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/950a09b0
Branch: refs/heads/branch-1
Commit: 950a09b03cf1a306ed5a2754c11457daf30e7d23
Parents: a8c8bfd
Author: Ashish Singhi
Authored: Fri Jun 3 18:46:41 2016 +0530
Committer: Ashish Singhi
Committed: Fri Jun 3 18:47:31 2016 +0530
--
.../replication/ReplicationPeersZKImpl.java | 6 ++
.../hbase/util/hbck/ReplicationChecker.java | 59 ++--
2 files changed, 61 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/950a09b0/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index c0c3f7e..076167e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -567,6 +567,12 @@ public class ReplicationPeersZKImpl extends
ReplicationStateZKBase implements Re
}
}
}
+ // Check for hfile-refs queue
+ if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+ && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+ }
} catch (KeeperException e) {
throw new ReplicationException("Could not check queues deleted with id="
+ peerId, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/950a09b0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index bf44a50..64212c9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
@InterfaceAudience.Private
public class ReplicationChecker {
private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+ private final ZooKeeperWatcher zkw;
private ErrorReporter errorReporter;
private ReplicationQueuesClient queuesClient;
private ReplicationPeers replicationPeers;
private ReplicationQueueDeletor queueDeletor;
// replicator with its queueIds for removed peers
private Map> undeletedQueueIds = new HashMap>();
-
+ // Set of un deleted hfile refs queue Ids
+ private Set undeletedHFileRefsQueueIds = new HashSet<>();
+ private final String hfileRefsZNode;
+
public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw,
HConnection connection,
ErrorReporter errorReporter) throws IOException {
try {
+ this.zkw = zkw;
this.errorReporter = errorReporter;
this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw,
conf, connection);
this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
} catch (ReplicationException e) {
throw new IOException("failed to construct ReplicationChecker", e);
}
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication",
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode,
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
}
public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
} catch (KeeperException ke) {
throw new IOException(ke);
}
+
+checkUnDeletedHFileRefsQueues(peerIds);
+ }
+
+ private void checkUnDeletedHFileRefsQueues(Set peerIds) throws
IOException {
+try {
+ if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+return;
+ }
+ List listOfPeers =
this.queuesClient.getAllPeersFromHFi
hbase git commit: HBASE-15888 Extend HBASE-12769 for bulk load data replication
Repository: hbase
Updated Branches:
refs/heads/master 72d3f2a86 -> 0cbce0762
HBASE-15888 Extend HBASE-12769 for bulk load data replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cbce076
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cbce076
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cbce076
Branch: refs/heads/master
Commit: 0cbce07626b77d9aa75a16f5e52c19428865dce7
Parents: 72d3f2a
Author: Ashish Singhi
Authored: Fri Jun 3 18:42:00 2016 +0530
Committer: Ashish Singhi
Committed: Fri Jun 3 18:42:00 2016 +0530
--
.../replication/ReplicationPeersZKImpl.java | 6 ++
.../hbase/util/hbck/ReplicationChecker.java | 59 ++--
2 files changed, 61 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 09d2100..15265d9 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -550,6 +550,12 @@ public class ReplicationPeersZKImpl extends
ReplicationStateZKBase implements Re
}
}
}
+ // Check for hfile-refs queue
+ if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+ && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+throw new ReplicationException("Undeleted queue for peerId: " + peerId
++ ", found in hfile-refs node path " + hfileRefsZNode);
+ }
} catch (KeeperException e) {
throw new ReplicationException("Could not check queues deleted with id="
+ peerId, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbce076/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 89f2557..e472558 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -51,16 +51,21 @@ import org.apache.zookeeper.KeeperException;
@InterfaceAudience.Private
public class ReplicationChecker {
private static final Log LOG = LogFactory.getLog(ReplicationChecker.class);
+ private final ZooKeeperWatcher zkw;
private final ErrorReporter errorReporter;
private final ReplicationQueuesClient queuesClient;
private final ReplicationPeers replicationPeers;
private final ReplicationQueueDeletor queueDeletor;
// replicator with its queueIds for removed peers
private final Map> undeletedQueueIds = new HashMap<>();
-
+ // replicator with its undeleted queueIds for removed peers in hfile-refs
queue
+ private Set undeletedHFileRefsQueueIds = new HashSet<>();
+ private final String hfileRefsZNode;
+
public ReplicationChecker(Configuration conf, ZooKeeperWatcher zkw,
ClusterConnection connection,
ErrorReporter errorReporter) throws IOException {
try {
+ this.zkw = zkw;
this.errorReporter = errorReporter;
this.queuesClient = ReplicationFactory.getReplicationQueuesClient(zkw,
conf, connection);
this.queuesClient.init();
@@ -71,6 +76,13 @@ public class ReplicationChecker {
} catch (ReplicationException e) {
throw new IOException("failed to construct ReplicationChecker", e);
}
+
+String replicationZNodeName = conf.get("zookeeper.znode.replication",
"replication");
+String replicationZNode = ZKUtil.joinZNode(this.zkw.baseZNode,
replicationZNodeName);
+String hfileRefsZNodeName =
+
conf.get(ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+
ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
}
public boolean hasUnDeletedQueues() {
@@ -103,13 +115,37 @@ public class ReplicationChecker {
} catch (KeeperException ke) {
throw new IOException(ke);
}
+
+checkUnDeletedHFileRefsQueues(peerIds);
+ }
+
+ private void checkUnDeletedHFileRefsQueues(Set peerIds) throws
IOException {
+try {
+ if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
+return;
