[hbase] branch branch-2.3 updated: HBASE-24665 MultiWAL : Avoid rolling of ALL WALs when one of the WAL needs a roll #2155
This is an automated email from the ASF dual-hosted git repository.
anoopsamjohn pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 7ef1aca HBASE-24665 MultiWAL : Avoid rolling of ALL WALs when one of
the WAL needs a roll #2155
7ef1aca is described below
commit 7ef1aca01af370e506986aa1c3769e68eb8bebdf
Author: WenFeiYi
AuthorDate: Tue Jul 28 12:37:37 2020 +0800
HBASE-24665 MultiWAL : Avoid rolling of ALL WALs when one of the WAL needs
a roll #2155
Co-authored-by: wen_yi
Signed-off-by: Anoop
Signed-off-by: Ramkrishna
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/regionserver/LogRoller.java | 4 +-
.../apache/hadoop/hbase/wal/AbstractWALRoller.java | 119 +
.../hadoop/hbase/regionserver/TestLogRoller.java | 117
3 files changed, 196 insertions(+), 44 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index f5049c9..992b117 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -63,7 +63,7 @@ public class LogRoller extends
AbstractWALRoller {
}
@VisibleForTesting
- Map getWalNeedsRoll() {
-return this.walNeedsRoll;
+ Map getWalNeedsRoll() {
+return this.wals;
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
index 3154c19..8aa8a63 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
@@ -20,12 +20,11 @@ package org.apache.hadoop.hbase.wal;
import java.io.Closeable;
import java.io.IOException;
import java.net.ConnectException;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
@@ -56,31 +55,31 @@ public abstract class AbstractWALRoller extends Thread
protected static final String WAL_ROLL_PERIOD_KEY =
"hbase.regionserver.logroll.period";
- protected final ConcurrentMap walNeedsRoll = new
ConcurrentHashMap<>();
+ protected final ConcurrentMap wals = new
ConcurrentHashMap<>();
protected final T abortable;
- private volatile long lastRollTime = System.currentTimeMillis();
// Period to roll log.
private final long rollPeriod;
private final int threadWakeFrequency;
// The interval to check low replication on hlog's pipeline
- private long checkLowReplicationInterval;
+ private final long checkLowReplicationInterval;
private volatile boolean running = true;
public void addWAL(WAL wal) {
// check without lock first
-if (walNeedsRoll.containsKey(wal)) {
+if (wals.containsKey(wal)) {
return;
}
// this is to avoid race between addWAL and requestRollAll.
synchronized (this) {
- if (walNeedsRoll.putIfAbsent(wal, Boolean.FALSE) == null) {
+ if (wals.putIfAbsent(wal, new RollController(wal)) == null) {
wal.registerWALActionsListener(new WALActionsListener() {
@Override
public void logRollRequested(WALActionsListener.RollRequestReason
reason) {
// TODO logs will contend with each other here, replace with e.g.
DelayedQueue
synchronized (AbstractWALRoller.this) {
- walNeedsRoll.put(wal, Boolean.TRUE);
+ RollController controller = wals.computeIfAbsent(wal, rc -> new
RollController(wal));
+ controller.requestRoll();
AbstractWALRoller.this.notifyAll();
}
}
@@ -91,9 +90,8 @@ public abstract class AbstractWALRoller
extends Thread
public void requestRollAll() {
synchronized (this) {
- List wals = new ArrayList(walNeedsRoll.keySet());
- for (WAL wal : wals) {
-walNeedsRoll.put(wal, Boolean.TRUE);
+ for (RollController controller : wals.values()) {
+controller.requestRoll();
}
notifyAll();
}
@@ -113,9 +111,9 @@ public abstract class AbstractWALRoller extends Thread
*/
private void checkLowReplication(long now) {
try {
- for (Entry entry : walNeedsRoll.entrySet()) {
+ for (Entry entry : wals.entrySet()) {
WAL wal = entry.getKey();
-boolean needRollAlready = entry.getValue();
+boolean needRollAlready =
[hbase] branch master updated: HBASE-11686 Shell code should create a binding / irb workspace instead of polluting the root namespace (#2141)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 7eff07d HBASE-11686 Shell code should create a binding / irb workspace instead of polluting the root namespace (#2141) 7eff07d is described below commit 7eff07d6bfc74080addb7b2ab4076a9e75d3175c Author: Elliot AuthorDate: Mon Jul 27 23:56:09 2020 -0400 HBASE-11686 Shell code should create a binding / irb workspace instead of polluting the root namespace (#2141) * HBASE-11686 Shell code should create a binding / irb workspace instead of polluting the root namespace - Refactor Shell.export_commands to define commands using ruby lambdas. Additionally, this change stores a reference to shell_inst in scope so that we no longer need to assume the existance of the variable @shell. - Add logic to Shell class for constructing an IRB workspace with its own binding and non-global receiver. This workspace is loaded with all HBase and IRB commands. - Create new method on Shell for evaluating input from an arbitrary IO instance within the created IRB workspace. This is based on work by Hsieh that was previously in bin/hirb.rb. This method is more generic and more testable. This single pattern can be used for both executing files and reading from stdin, therefore reducing complexity. - Move special 'help' and 'tools' command definitions to shell.rb. These commands are tightly linked with an instance of the shell, so it is easiest to have them defined together. - Remove all global includes of HBaseConstants from ruby test files. Before this change, tests were loading these constants into the top level, which could cause tests to pass that should really fail. - Try to reduce the number of places that constants are included. I think it's best to reference each ruby constant's full name, but where that would cause a big diff I instead moved the include to the innermost Module or Class. - Update docs and comments - Remove unneccessary includes - Add shell --top-level-cmds compatibility flag. Since this PR removes all the HBase symbols from the top-level receiver (ie. main Object), it is possible (albeit unlikely) that this will break operator scripts. This flag will export all the commands at the top-level like the shell previously did. * HBASE-11686 Light refactoring with added unit tests - Fixes some constants references by admin test 2 - Install HBase commands as singleton methods on recevier instances so that multiple receivers may exist. - Rename new flag from --top-level-cmds to --top-level-defs to be more semantically accurate. - Create new helper method Shell::Shell#export_all to install @hbase, @shell, constants, and all hbase commands to a target receiver. As a result, the HBaseReceiver became very simple and could be moved to shell.rb. - Add unit tests for Shell::Shell#eval_io and Shell::Shell#export_all - Add @hbase and @shell to hbase-shell IRB workspace - Fix robocop issues within patch * Typo s/is/if/ --- bin/hirb.rb| 87 -- hbase-shell/src/main/ruby/hbase/admin.rb | 3 +- hbase-shell/src/main/ruby/hbase/quotas.rb | 3 + hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb | 8 +- hbase-shell/src/main/ruby/hbase/security.rb| 2 - hbase-shell/src/main/ruby/hbase/taskmonitor.rb | 2 - hbase-shell/src/main/ruby/hbase_constants.rb | 4 +- hbase-shell/src/main/ruby/shell.rb | 130 +++-- .../src/main/ruby/shell/commands/clone_snapshot.rb | 2 +- .../src/main/ruby/shell/commands/describe.rb | 2 +- .../main/ruby/shell/commands/describe_namespace.rb | 2 +- .../src/main/ruby/shell/commands/list_regions.rb | 17 ++- .../main/ruby/shell/commands/restore_snapshot.rb | 2 +- .../src/main/ruby/shell/commands/set_quota.rb | 29 +++-- hbase-shell/src/test/ruby/hbase/admin2_test.rb | 7 +- hbase-shell/src/test/ruby/hbase/admin_test.rb | 16 ++- .../ruby/hbase/list_regions_test_no_cluster.rb | 3 +- hbase-shell/src/test/ruby/hbase/quotas_test.rb | 4 +- .../src/test/ruby/hbase/quotas_test_no_cluster.rb | 3 +- .../src/test/ruby/hbase/replication_admin_test.rb | 2 +- .../src/test/ruby/hbase/security_admin_test.rb | 3 +- hbase-shell/src/test/ruby/hbase/table_test.rb | 6 +- .../test/ruby/hbase/test_connection_no_cluster.rb | 3 +- .../ruby/hbase/visibility_labels_admin_test.rb | 3 +- hbase-shell/src/test/ruby/shell/converter_test.rb | 3 +- .../src/test/ruby/shell/list_procedures_test.rb| 3 +-
[hbase] branch master updated: HBASE-24770 Reimplement the Constraints API and revisit the IA annotations on related classes (#2140)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 7e6e7a7 HBASE-24770 Reimplement the Constraints API and revisit the
IA annotations on related classes (#2140)
7e6e7a7 is described below
commit 7e6e7a7051f71c4f554c53d756538cb753cdfc66
Author: Duo Zhang
AuthorDate: Tue Jul 28 09:24:55 2020 +0800
HBASE-24770 Reimplement the Constraints API and revisit the IA annotations
on related classes (#2140)
Signed-off-by: stack
---
.../hbase/client/TableDescriptorBuilder.java | 20 +-
.../apache/hadoop/hbase/constraint/Constraint.java | 78 ++-
.../hadoop/hbase/constraint/Constraints.java | 604 +++--
.../hadoop/hbase/constraint/package-info.java | 465
.../hadoop/hbase/constraint/TestConstraint.java| 95 ++--
.../hadoop/hbase/constraint/TestConstraints.java | 136 ++---
6 files changed, 609 insertions(+), 789 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
index e929811..bf591a1 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
@@ -32,11 +32,11 @@ import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
@@ -380,6 +380,10 @@ public class TableDescriptorBuilder {
return this;
}
+ public boolean hasCoprocessor(String classNameToMatch) {
+return desc.hasCoprocessor(classNameToMatch);
+ }
+
public TableDescriptorBuilder setColumnFamily(final ColumnFamilyDescriptor
family) {
desc.setColumnFamily(Objects.requireNonNull(family));
return this;
@@ -411,6 +415,16 @@ public class TableDescriptorBuilder {
return this;
}
+ public TableDescriptorBuilder removeValue(BiPredicate
predicate) {
+List toRemove =
+ desc.getValues().entrySet().stream().filter(e ->
predicate.test(e.getKey(), e.getValue()))
+.map(Map.Entry::getKey).collect(Collectors.toList());
+for (Bytes key : toRemove) {
+ removeValue(key);
+}
+return this;
+ }
+
public TableDescriptorBuilder removeColumnFamily(final byte[] name) {
desc.removeColumnFamily(name);
return this;
@@ -531,6 +545,10 @@ public class TableDescriptorBuilder {
return this;
}
+ public String getValue(String key) {
+return desc.getValue(key);
+ }
+
/**
* Sets replication scope all & only the columns already in the builder.
Columns added later won't
* be backfilled with replication scope.
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java
index 4a63ec1..c0c4b60 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/constraint/Constraint.java
@@ -22,45 +22,39 @@ import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.hbase.client.Put;
/**
- * Apply a {@link Constraint} (in traditional database terminology) to a
HTable.
- * Any number of {@link Constraint Constraints} can be added to the table, in
- * any order.
- *
+ * Apply a {@link Constraint} (in traditional database terminology) to a
Table. Any number of
+ * {@link Constraint Constraints} can be added to the table, in any order.
+ *
* A {@link Constraint} must be added to a table before the table is loaded via
- * {@link Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class[])}
or
- * {@link Constraints#add(org.apache.hadoop.hbase.HTableDescriptor,
- * org.apache.hadoop.hbase.util.Pair...)}
- * (if you want to add a configuration with the {@link Constraint}).
Constraints
- * will be run in the order that they are added. Further, a Constraint will be
- * configured before it is run (on load).
- *
- * See {@link
Constraints#enableConstraint(org.apache.hadoop.hbase.HTableDescriptor, Class)}
and
- * {@link
Constraints#disableConstraint(org.apache.hadoop.hbase.HTableDescriptor, Class)}
for
- * enabling/disabling of a given {@link Constraint} after it has been added.
- *
+ * {@link
Constraints#add(org.apache.hadoop.hbase.client.TableDescriptorBuilder,
Class...)} or
+ * {@link
Constraints#add(org.apache.hadoop.hbase.client.TableDescriptorBuilder,
[hbase] branch branch-1 updated: HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
This is an automated email from the ASF dual-hosted git repository.
brfrn169 pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new b0d49ae HBASE-24775 [hbtop] StoreFile size should be rounded off
(#2144)
b0d49ae is described below
commit b0d49aebeaf05a5cc045eeda7b819cccf7cbee9d
Author: Toshihiro Suzuki
AuthorDate: Tue Jul 28 08:14:45 2020 +0900
HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
index db7d22f..bbfe508 100644
---
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
+++
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
@@ -174,9 +174,12 @@ public final class FieldValue implements
Comparable {
case INTEGER:
case LONG:
case FLOAT:
- case SIZE:
return value.toString();
+ case SIZE:
+Size size = (Size) value;
+return String.format("%.1f", size.get()) +
size.getUnit().getSimpleName();
+
case PERCENT:
return String.format("%.2f", (Float) value) + "%";
[hbase] branch branch-2.2 updated: HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
This is an automated email from the ASF dual-hosted git repository.
brfrn169 pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new c774e36 HBASE-24775 [hbtop] StoreFile size should be rounded off
(#2144)
c774e36 is described below
commit c774e36d25c15c1c9f09fe5669a932b810cf3c40
Author: Toshihiro Suzuki
AuthorDate: Tue Jul 28 08:14:45 2020 +0900
HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
index 6150df9..086dadc 100644
---
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
+++
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
@@ -175,9 +175,12 @@ public final class FieldValue implements
Comparable {
case INTEGER:
case LONG:
case FLOAT:
- case SIZE:
return value.toString();
+ case SIZE:
+Size size = (Size) value;
+return String.format("%.1f", size.get()) +
size.getUnit().getSimpleName();
+
case PERCENT:
return String.format("%.2f", (Float) value) + "%";
[hbase] branch branch-2.3 updated: HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
This is an automated email from the ASF dual-hosted git repository.
brfrn169 pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new ece0792 HBASE-24775 [hbtop] StoreFile size should be rounded off
(#2144)
ece0792 is described below
commit ece0792bf4f7e04a04a83536ef279e62398219b3
Author: Toshihiro Suzuki
AuthorDate: Tue Jul 28 08:14:45 2020 +0900
HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
index 6150df9..086dadc 100644
---
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
+++
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
@@ -175,9 +175,12 @@ public final class FieldValue implements
Comparable {
case INTEGER:
case LONG:
case FLOAT:
- case SIZE:
return value.toString();
+ case SIZE:
+Size size = (Size) value;
+return String.format("%.1f", size.get()) +
size.getUnit().getSimpleName();
+
case PERCENT:
return String.format("%.2f", (Float) value) + "%";
[hbase] branch branch-2 updated: HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
This is an automated email from the ASF dual-hosted git repository.
brfrn169 pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 2720a9d HBASE-24775 [hbtop] StoreFile size should be rounded off
(#2144)
2720a9d is described below
commit 2720a9d93e134f0c3b752446069c174a79e673a1
Author: Toshihiro Suzuki
AuthorDate: Tue Jul 28 08:14:45 2020 +0900
HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
index 6150df9..086dadc 100644
---
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
+++
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
@@ -175,9 +175,12 @@ public final class FieldValue implements
Comparable {
case INTEGER:
case LONG:
case FLOAT:
- case SIZE:
return value.toString();
+ case SIZE:
+Size size = (Size) value;
+return String.format("%.1f", size.get()) +
size.getUnit().getSimpleName();
+
case PERCENT:
return String.format("%.2f", (Float) value) + "%";
[hbase] branch master updated: HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
This is an automated email from the ASF dual-hosted git repository.
brfrn169 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 477debd HBASE-24775 [hbtop] StoreFile size should be rounded off
(#2144)
477debd is described below
commit 477debdc74c26275131884d91e177cd5f764bdd9
Author: Toshihiro Suzuki
AuthorDate: Tue Jul 28 08:14:45 2020 +0900
HBASE-24775 [hbtop] StoreFile size should be rounded off (#2144)
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
index 6150df9..086dadc 100644
---
a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
+++
b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java
@@ -175,9 +175,12 @@ public final class FieldValue implements
Comparable {
case INTEGER:
case LONG:
case FLOAT:
- case SIZE:
return value.toString();
+ case SIZE:
+Size size = (Size) value;
+return String.format("%.1f", size.get()) +
size.getUnit().getSimpleName();
+
case PERCENT:
return String.format("%.2f", (Float) value) + "%";
[hbase] branch HBASE-15519 created (now d07c9b0)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch HBASE-15519 in repository https://gitbox.apache.org/repos/asf/hbase.git. at d07c9b0 HBASE-24767 Change default to false for HBASE-15519 per-user metrics Set hbase.regionserver.user.metrics.enabled default to false; i.e. off. No new revisions were added by this update.
[hbase] branch HBASE-24767 created (now d07c9b0)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch HBASE-24767 in repository https://gitbox.apache.org/repos/asf/hbase.git. at d07c9b0 HBASE-24767 Change default to false for HBASE-15519 per-user metrics Set hbase.regionserver.user.metrics.enabled default to false; i.e. off. This branch includes the following new commits: new d07c9b0 HBASE-24767 Change default to false for HBASE-15519 per-user metrics Set hbase.regionserver.user.metrics.enabled default to false; i.e. off. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[hbase] 01/01: HBASE-24767 Change default to false for HBASE-15519 per-user metrics Set hbase.regionserver.user.metrics.enabled default to false; i.e. off.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch HBASE-24767
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d07c9b060b3a6f41b39be6fe708663677e11b30d
Author: stack
AuthorDate: Thu Jul 23 17:05:00 2020 -0700
HBASE-24767 Change default to false for HBASE-15519 per-user metrics Set
hbase.regionserver.user.metrics.enabled default to false; i.e. off.
---
.../apache/hadoop/hbase/regionserver/MetricsUserAggregateFactory.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsUserAggregateFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsUserAggregateFactory.java
index 38e440b..f766e91 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsUserAggregateFactory.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsUserAggregateFactory.java
@@ -28,7 +28,7 @@ public class MetricsUserAggregateFactory {
}
public static final String METRIC_USER_ENABLED_CONF =
"hbase.regionserver.user.metrics.enabled";
- public static final boolean DEFAULT_METRIC_USER_ENABLED_CONF = true;
+ public static final boolean DEFAULT_METRIC_USER_ENABLED_CONF = false;
public static MetricsUserAggregate getMetricsUserAggregate(Configuration
conf) {
if (conf.getBoolean(METRIC_USER_ENABLED_CONF,
DEFAULT_METRIC_USER_ENABLED_CONF)) {
[hbase] branch branch-1 updated: HBASE-24757 : ReplicationSink should limit row count in batch mutation based on hbase.rpc.rows.warning.threshold (#2146)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new dd4417a HBASE-24757 : ReplicationSink should limit row count in batch
mutation based on hbase.rpc.rows.warning.threshold (#2146)
dd4417a is described below
commit dd4417a9e23172eb6535a6751c3694a9c7d49d68
Author: Viraj Jasani
AuthorDate: Tue Jul 28 01:10:02 2020 +0530
HBASE-24757 : ReplicationSink should limit row count in batch mutation
based on hbase.rpc.rows.warning.threshold (#2146)
Closes #2127
Signed-off-by: stack
---
.../java/org/apache/hadoop/hbase/HConstants.java | 10 +
.../hadoop/hbase/regionserver/RSRpcServices.java | 12 +-
.../replication/regionserver/Replication.java | 2 +-
.../replication/regionserver/ReplicationSink.java | 34 +++
.../hbase/regionserver/TestMultiLogThreshold.java | 5 ++-
.../regionserver/TestReplicationSink.java | 49 ++
6 files changed, 83 insertions(+), 29 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 94d977b..28b2d1c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1390,6 +1390,16 @@ public final class HConstants {
"hbase.master.executor.logreplayops.threads";
public static final int MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT = 10;
+ /**
+ * Number of rows in a batch operation above which a warning will be logged.
+ */
+ public static final String BATCH_ROWS_THRESHOLD_NAME =
"hbase.rpc.rows.warning.threshold";
+
+ /**
+ * Default value of {@link #BATCH_ROWS_THRESHOLD_NAME}
+ */
+ public static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
+
private HConstants() {
// Can't be instantiated with this ctor.
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 34f322c..86bbe89 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -231,15 +231,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
*/
private static final long
DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10;
- /**
- * Number of rows in a batch operation above which a warning will be logged.
- */
- static final String BATCH_ROWS_THRESHOLD_NAME =
"hbase.rpc.rows.warning.threshold";
- /**
- * Default value of {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME}
- */
- static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
-
/*
* Whether to reject rows with size > threshold defined by
* {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME}
@@ -1128,7 +1119,8 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
RSRpcServices(HRegionServer rs, LogDelegate ld) throws IOException {
this.ld = ld;
regionServer = rs;
-rowSizeWarnThreshold = rs.conf.getInt(BATCH_ROWS_THRESHOLD_NAME,
BATCH_ROWS_THRESHOLD_DEFAULT);
+rowSizeWarnThreshold = rs.conf.getInt(
+ HConstants.BATCH_ROWS_THRESHOLD_NAME,
HConstants.BATCH_ROWS_THRESHOLD_DEFAULT);
RpcSchedulerFactory rpcSchedulerFactory;
rejectRowsWithSizeOverThreshold = rs.conf
.getBoolean(REJECT_BATCH_ROWS_OVER_THRESHOLD,
DEFAULT_REJECT_BATCH_ROWS_OVER_THRESHOLD);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index d6f48b9..03cd86b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -245,7 +245,7 @@ public class Replication extends WALActionsListener.Base
implements
} catch (ReplicationException e) {
throw new IOException(e);
}
- this.replicationSink = new ReplicationSink(this.conf, this.server);
+ this.replicationSink = new ReplicationSink(this.conf);
this.scheduleThreadPool.scheduleAtFixedRate(
new ReplicationStatisticsThread(this.replicationSink,
this.replicationManager),
statsThreadPeriod, statsThreadPeriod, TimeUnit.SECONDS);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 9143f3d..34cb867 100644
---
[hbase] branch branch-1 updated: HBASE-24777 InfoServer support ipv6 host and port
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new b154f20 HBASE-24777 InfoServer support ipv6 host and port
b154f20 is described below
commit b154f20c4dba95aebdfc845282f1c8543f82799e
Author: YeChao Chen
AuthorDate: Mon Jul 27 21:25:57 2020 +0530
HBASE-24777 InfoServer support ipv6 host and port
Closes #2147
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
index 0f6c3dd..9dbd766 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.http;
+import com.google.common.net.HostAndPort;
+
import java.io.IOException;
import java.net.URI;
@@ -28,6 +30,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal
* is to serve up status information for the server.
@@ -60,8 +63,8 @@ public class InfoServer {
new org.apache.hadoop.hbase.http.HttpServer.Builder();
builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() +
-bindAddress + ":" +
-port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
+HostAndPort.fromParts(bindAddress, port).toString()))
+ .setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
String logDir = System.getProperty("hbase.log.dir");
if (logDir != null) {
builder.setLogDir(logDir);
[hbase] branch branch-1 updated: HBASE-24777 InfoServer support ipv6 host and port
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new b154f20 HBASE-24777 InfoServer support ipv6 host and port
b154f20 is described below
commit b154f20c4dba95aebdfc845282f1c8543f82799e
Author: YeChao Chen
AuthorDate: Mon Jul 27 21:25:57 2020 +0530
HBASE-24777 InfoServer support ipv6 host and port
Closes #2147
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
index 0f6c3dd..9dbd766 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.http;
+import com.google.common.net.HostAndPort;
+
import java.io.IOException;
import java.net.URI;
@@ -28,6 +30,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal
* is to serve up status information for the server.
@@ -60,8 +63,8 @@ public class InfoServer {
new org.apache.hadoop.hbase.http.HttpServer.Builder();
builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() +
-bindAddress + ":" +
-port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
+HostAndPort.fromParts(bindAddress, port).toString()))
+ .setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
String logDir = System.getProperty("hbase.log.dir");
if (logDir != null) {
builder.setLogDir(logDir);
[hbase] branch branch-2.2 updated: HBASE-24777 InfoServer support ipv6 host and port
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 5eb4b87 HBASE-24777 InfoServer support ipv6 host and port
5eb4b87 is described below
commit 5eb4b87e7f3f1f66ba0689c7a0be614b1e64b709
Author: YeChao Chen
AuthorDate: Mon Jul 27 21:25:57 2020 +0530
HBASE-24777 InfoServer support ipv6 host and port
Closes #2147
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
index 586a3f3..fdd87fd 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal
* is to serve up status information for the server.
@@ -64,8 +66,8 @@ public class InfoServer {
new org.apache.hadoop.hbase.http.HttpServer.Builder();
builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() +
-bindAddress + ":" +
-port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
+HostAndPort.fromParts(bindAddress, port).toString()))
+ .setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
String logDir = System.getProperty("hbase.log.dir");
if (logDir != null) {
builder.setLogDir(logDir);
[hbase] branch branch-2 updated: HBASE-24632 Enable procedure-based log splitting as default in hbase3 Add deprecation of 'classic' zk-based WAL splitter.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new edb4cd5 HBASE-24632 Enable procedure-based log splitting as default
in hbase3 Add deprecation of 'classic' zk-based WAL splitter.
edb4cd5 is described below
commit edb4cd534c6ad96745009b2bcc8587d52d1657a2
Author: stack
AuthorDate: Tue Jun 30 20:53:36 2020 -0700
HBASE-24632 Enable procedure-based log splitting as default in hbase3 Add
deprecation of 'classic' zk-based WAL splitter.
Also fix three bugs:
* We were trying to delete non-empty directory; weren't doing
accounting for meta WALs where meta had moved off the server
(successfully)
* We were deleting split WALs rather than archiving them.
* We were not handling corrupt files.
Deprecations and removal of tests of old system.
---
.../java/org/apache/hadoop/hbase/HConstants.java | 11 +-
.../org/apache/hadoop/hbase/SplitLogCounters.java | 5 +
.../java/org/apache/hadoop/hbase/SplitLogTask.java | 3 +
.../coordination/SplitLogManagerCoordination.java | 3 +
.../coordination/SplitLogWorkerCoordination.java | 10 +-
.../coordination/ZkCoordinatedStateManager.java| 3 +
.../hadoop/hbase/master/MasterRpcServices.java | 7 +
.../hadoop/hbase/master/MasterWalManager.java | 34 ++-
.../hbase/master/MetricsMasterWrapperImpl.java | 3 +
.../hadoop/hbase/master/SplitLogManager.java | 3 +
.../hadoop/hbase/master/SplitWALManager.java | 41 ++-
.../master/procedure/ServerCrashProcedure.java | 55 ++--
.../master/procedure/SplitWALRemoteProcedure.java | 2 +-
.../hadoop/hbase/regionserver/SplitLogWorker.java | 8 +-
.../regionserver/handler/WALSplitterHandler.java | 5 +-
.../org/apache/hadoop/hbase/wal/WALSplitUtil.java | 105 +++-
.../org/apache/hadoop/hbase/wal/WALSplitter.java | 290 -
.../hadoop/hbase/master/AbstractTestDLS.java | 239 +
.../master/assignment/MockMasterServices.java | 22 +-
.../hbase/regionserver/TestCleanupMetaWAL.java | 2 +-
.../hadoop/hbase/wal/TestWALReaderOnSecureWAL.java | 22 +-
.../org/apache/hadoop/hbase/wal/TestWALSplit.java | 56 ++--
.../hadoop/hbase/zookeeper/MetaTableLocator.java | 4 +-
.../apache/hadoop/hbase/zookeeper/ZKSplitLog.java | 3 +
24 files changed, 392 insertions(+), 544 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 1eea08b..16bee93 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1480,9 +1480,18 @@ public final class HConstants {
public static final String HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL =
"hbase.client.fast.fail.interceptor.impl";
+ /**
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by
procedure-based
+ * distributed WAL splitter; see SplitWALManager.
+ */
+ @Deprecated
public static final String HBASE_SPLIT_WAL_COORDINATED_BY_ZK =
"hbase.split.wal.zk.coordinated";
- public static final boolean DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK = true;
+ /**
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0.
+ */
+ @Deprecated
+ public static final boolean DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK = false;
public static final String HBASE_SPLIT_WAL_MAX_SPLITTER =
"hbase.regionserver.wal.max.splitters";
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogCounters.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogCounters.java
index 6be1131..443c8d2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogCounters.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogCounters.java
@@ -25,9 +25,14 @@ import org.apache.yetus.audience.InterfaceAudience;
/**
* Counters kept by the distributed WAL split log process.
* Used by master and regionserver packages.
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by
procedure-based
+ * distributed WAL splitter, see SplitWALManager
*/
+@Deprecated
@InterfaceAudience.Private
public class SplitLogCounters {
+ private SplitLogCounters() {}
+
//Spnager counters
public final static LongAdder tot_mgr_log_split_batch_start = new
LongAdder();
public final static LongAdder tot_mgr_log_split_batch_success = new
LongAdder();
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
index dd4eb93..ca07fcb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
+++
[hbase] branch branch-2.3 updated: HBASE-24777 InfoServer support ipv6 host and port
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 93d9246 HBASE-24777 InfoServer support ipv6 host and port
93d9246 is described below
commit 93d924615fcbabf43973c216d1cd061092439326
Author: YeChao Chen
AuthorDate: Mon Jul 27 20:41:49 2020 +0530
HBASE-24777 InfoServer support ipv6 host and port
Closes #2147
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
index 6a1ffbc..3dd3877 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal
* is to serve up status information for the server.
@@ -61,8 +63,8 @@ public class InfoServer {
new org.apache.hadoop.hbase.http.HttpServer.Builder();
builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() +
- bindAddress + ":" +
- port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
+ HostAndPort.fromParts(bindAddress,port).toString())).
+setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
String logDir = System.getProperty("hbase.log.dir");
if (logDir != null) {
builder.setLogDir(logDir);
[hbase] branch branch-2 updated: HBASE-24777 InfoServer support ipv6 host and port
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 8f31b12 HBASE-24777 InfoServer support ipv6 host and port
8f31b12 is described below
commit 8f31b12ba4fa78fb7673221c020c02c244d9ba86
Author: YeChao Chen
AuthorDate: Mon Jul 27 20:41:49 2020 +0530
HBASE-24777 InfoServer support ipv6 host and port
Closes #2147
Signed-off-by: Duo Zhang
Signed-off-by: Viraj Jasani
---
.../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
index 6a1ffbc..3dd3877 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal
* is to serve up status information for the server.
@@ -61,8 +63,8 @@ public class InfoServer {
new org.apache.hadoop.hbase.http.HttpServer.Builder();
builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() +
- bindAddress + ":" +
- port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
+ HostAndPort.fromParts(bindAddress,port).toString())).
+setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c);
String logDir = System.getProperty("hbase.log.dir");
if (logDir != null) {
builder.setLogDir(logDir);
[hbase] branch master updated (8c0d7fa -> 82d0990)
This is an automated email from the ASF dual-hosted git repository. vjasani pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from 8c0d7fa HBASE-24758 Avoid flooding replication source RSes logs when no sinks… (#2118) add 82d0990 HBASE-24777 InfoServer support ipv6 host and port No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/hadoop/hbase/http/InfoServer.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-)
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 4bc106b INFRA-10751 Empty commit 4bc106b is described below commit 4bc106ba09f17baf27d8e37f7563acaca5d55924 Author: jenkins AuthorDate: Mon Jul 27 14:50:58 2020 + INFRA-10751 Empty commit
[hbase] branch branch-2.2 updated: HBASE-24757 : ReplicationSink should limit row count in batch mutation based on hbase.rpc.rows.warning.threshold (#2139)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 603d2b6 HBASE-24757 : ReplicationSink should limit row count in batch
mutation based on hbase.rpc.rows.warning.threshold (#2139)
603d2b6 is described below
commit 603d2b629aeb749b2ca009dbf54149d7240bb269
Author: Viraj Jasani
AuthorDate: Mon Jul 27 20:15:51 2020 +0530
HBASE-24757 : ReplicationSink should limit row count in batch mutation
based on hbase.rpc.rows.warning.threshold (#2139)
Closes #2127
Signed-off-by: stack
---
.../java/org/apache/hadoop/hbase/HConstants.java | 10
.../hadoop/hbase/regionserver/RSRpcServices.java | 12 +
.../replication/regionserver/Replication.java | 2 +-
.../replication/regionserver/ReplicationSink.java | 60 --
.../hbase/regionserver/TestMultiLogThreshold.java | 5 +-
.../regionserver/TestReplicationSink.java | 49 +++---
.../regionserver/TestWALEntrySinkFilter.java | 2 +-
7 files changed, 91 insertions(+), 49 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 05366a3..d1537fa 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1493,6 +1493,16 @@ public final class HConstants {
"hbase.master.executor.logreplayops.threads";
public static final int MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT = 10;
+ /**
+ * Number of rows in a batch operation above which a warning will be logged.
+ */
+ public static final String BATCH_ROWS_THRESHOLD_NAME =
"hbase.rpc.rows.warning.threshold";
+
+ /**
+ * Default value of {@link #BATCH_ROWS_THRESHOLD_NAME}
+ */
+ public static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
+
private HConstants() {
// Can't be instantiated with this ctor.
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 9a02831..0819423 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -276,15 +276,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
*/
private static final long
DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10;
- /**
- * Number of rows in a batch operation above which a warning will be logged.
- */
- static final String BATCH_ROWS_THRESHOLD_NAME =
"hbase.rpc.rows.warning.threshold";
- /**
- * Default value of {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME}
- */
- static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
-
protected static final String RESERVOIR_ENABLED_KEY =
"hbase.ipc.server.reservoir.enabled";
// Request counter. (Includes requests that are not serviced by regions.)
@@ -1229,7 +1220,8 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
RSRpcServices(HRegionServer rs, LogDelegate ld) throws IOException {
this.ld = ld;
regionServer = rs;
-rowSizeWarnThreshold = rs.conf.getInt(BATCH_ROWS_THRESHOLD_NAME,
BATCH_ROWS_THRESHOLD_DEFAULT);
+rowSizeWarnThreshold = rs.conf.getInt(HConstants.BATCH_ROWS_THRESHOLD_NAME,
+ HConstants.BATCH_ROWS_THRESHOLD_DEFAULT);
RpcSchedulerFactory rpcSchedulerFactory;
try {
rpcSchedulerFactory =
getRpcSchedulerFactoryClass().asSubclass(RpcSchedulerFactory.class)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 6c46a85..752cfb8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -187,7 +187,7 @@ public class Replication implements
ReplicationSourceService, ReplicationSinkSer
@Override
public void startReplicationService() throws IOException {
this.replicationManager.init();
-this.replicationSink = new ReplicationSink(this.conf, this.server);
+this.replicationSink = new ReplicationSink(this.conf);
this.scheduleThreadPool.scheduleAtFixedRate(
new ReplicationStatisticsTask(this.replicationSink,
this.replicationManager),
statsThreadPeriod, statsThreadPeriod, TimeUnit.SECONDS);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
[hbase] branch branch-2.3 updated: HBASE-24758 Avoid flooding replication source RSes logs when no sinks… (#2118)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 1e6f108 HBASE-24758 Avoid flooding replication source RSes logs when
no sinks… (#2118)
1e6f108 is described below
commit 1e6f108bea65c910f9fd1b754768297cacc26f6d
Author: Wellington Ramos Chevreuil
AuthorDate: Mon Jul 27 10:08:13 2020 +0100
HBASE-24758 Avoid flooding replication source RSes logs when no sinks…
(#2118)
Signed-off-by: Josh Elser
Signed-off-by: Viraj Jasani
(cherry picked from commit 8c0d7fa5b8971de8bc7062675ba96a6091263776)
---
.../hadoop/hbase/replication/HBaseReplicationEndpoint.java | 4 ++--
.../regionserver/HBaseInterClusterReplicationEndpoint.java | 12 ++--
.../replication/regionserver/ReplicationSinkManager.java | 3 +++
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
index 1ca70ad..3cde0d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
@@ -168,8 +168,8 @@ public abstract class HBaseReplicationEndpoint extends
BaseReplicationEndpoint
}
/**
- * Get a list of all the addresses of all the region servers
- * for this peer cluster
+ * Get a list of all the addresses of all the available region servers
+ * for this peer cluster, or an empty list if no region servers available at
peer cluster.
* @return list of addresses
*/
// Synchronize peer cluster connection attempts to avoid races and rate
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index 9539d30..4fb6146 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -128,6 +128,8 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
private boolean dropOnDeletedTables;
private boolean dropOnDeletedColumnFamilies;
private boolean isSerial = false;
+ //Initialising as 0 to guarantee at least one logging message
+ private long lastSinkFetchTime = 0;
/*
* Some implementations of HBaseInterClusterReplicationEndpoint may require
instantiating
@@ -518,8 +520,14 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
int numSinks = replicationSinkMgr.getNumSinks();
if (numSinks == 0) {
- LOG.warn("{} No replication sinks found, returning without replicating. "
-+ "The source should retry with the same set of edits.", logPeerId());
+ if((System.currentTimeMillis() - lastSinkFetchTime) >=
(maxRetriesMultiplier*1000)) {
+LOG.warn(
+ "No replication sinks found, returning without replicating. "
++ "The source should retry with the same set of edits. Not logging
this again for "
++ "the next {} seconds.", maxRetriesMultiplier);
+lastSinkFetchTime = System.currentTimeMillis();
+ }
+ sleepForRetries("No sinks available at peer", sleepMultiplier);
return false;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
index 3cd7884..4dc5b33 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
@@ -160,6 +160,9 @@ public class ReplicationSinkManager {
*/
public synchronized void chooseSinks() {
List slaveAddresses = endpoint.getRegionServers();
+if(slaveAddresses.isEmpty()){
+ LOG.warn("No sinks available at peer. Will not be able to replicate");
+}
Collections.shuffle(slaveAddresses, random);
int numSinks = (int) Math.ceil(slaveAddresses.size() * ratio);
sinks = slaveAddresses.subList(0, numSinks);
[hbase] branch branch-2 updated: HBASE-24758 Avoid flooding replication source RSes logs when no sinks… (#2118)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new fce52fe HBASE-24758 Avoid flooding replication source RSes logs when
no sinks… (#2118)
fce52fe is described below
commit fce52fe6bb320e6c8fd4e6c1073569d686a0ffb3
Author: Wellington Ramos Chevreuil
AuthorDate: Mon Jul 27 10:08:13 2020 +0100
HBASE-24758 Avoid flooding replication source RSes logs when no sinks…
(#2118)
Signed-off-by: Josh Elser
Signed-off-by: Viraj Jasani
(cherry picked from commit 8c0d7fa5b8971de8bc7062675ba96a6091263776)
---
.../hadoop/hbase/replication/HBaseReplicationEndpoint.java | 4 ++--
.../regionserver/HBaseInterClusterReplicationEndpoint.java | 12 ++--
.../replication/regionserver/ReplicationSinkManager.java | 3 +++
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
index 1ca70ad..3cde0d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
@@ -168,8 +168,8 @@ public abstract class HBaseReplicationEndpoint extends
BaseReplicationEndpoint
}
/**
- * Get a list of all the addresses of all the region servers
- * for this peer cluster
+ * Get a list of all the addresses of all the available region servers
+ * for this peer cluster, or an empty list if no region servers available at
peer cluster.
* @return list of addresses
*/
// Synchronize peer cluster connection attempts to avoid races and rate
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index 9539d30..4fb6146 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -128,6 +128,8 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
private boolean dropOnDeletedTables;
private boolean dropOnDeletedColumnFamilies;
private boolean isSerial = false;
+ //Initialising as 0 to guarantee at least one logging message
+ private long lastSinkFetchTime = 0;
/*
* Some implementations of HBaseInterClusterReplicationEndpoint may require
instantiating
@@ -518,8 +520,14 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
int numSinks = replicationSinkMgr.getNumSinks();
if (numSinks == 0) {
- LOG.warn("{} No replication sinks found, returning without replicating. "
-+ "The source should retry with the same set of edits.", logPeerId());
+ if((System.currentTimeMillis() - lastSinkFetchTime) >=
(maxRetriesMultiplier*1000)) {
+LOG.warn(
+ "No replication sinks found, returning without replicating. "
++ "The source should retry with the same set of edits. Not logging
this again for "
++ "the next {} seconds.", maxRetriesMultiplier);
+lastSinkFetchTime = System.currentTimeMillis();
+ }
+ sleepForRetries("No sinks available at peer", sleepMultiplier);
return false;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
index 3cd7884..4dc5b33 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
@@ -160,6 +160,9 @@ public class ReplicationSinkManager {
*/
public synchronized void chooseSinks() {
List slaveAddresses = endpoint.getRegionServers();
+if(slaveAddresses.isEmpty()){
+ LOG.warn("No sinks available at peer. Will not be able to replicate");
+}
Collections.shuffle(slaveAddresses, random);
int numSinks = (int) Math.ceil(slaveAddresses.size() * ratio);
sinks = slaveAddresses.subList(0, numSinks);
[hbase] branch branch-2 updated: HBASE-24665 (#2150)
This is an automated email from the ASF dual-hosted git repository.
anoopsamjohn pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 2bb76c0 HBASE-24665 (#2150)
2bb76c0 is described below
commit 2bb76c0a51c87eeab4508c6bdfdd3c96cf0b26d6
Author: WenFeiYi
AuthorDate: Mon Jul 27 19:09:49 2020 +0800
HBASE-24665 (#2150)
Co-authored-by: wen_yi
Signed-off-by: Anoop
Signed-off-by: Ramkrishna
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/regionserver/LogRoller.java | 4 +-
.../apache/hadoop/hbase/wal/AbstractWALRoller.java | 119 +
.../hadoop/hbase/regionserver/TestLogRoller.java | 117
3 files changed, 196 insertions(+), 44 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 58ac82e..cbab595 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -65,7 +65,7 @@ public class LogRoller extends
AbstractWALRoller {
}
@VisibleForTesting
- Map getWalNeedsRoll() {
-return this.walNeedsRoll;
+ Map getWalNeedsRoll() {
+return this.wals;
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
index 6999020..9263c0f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
@@ -20,13 +20,13 @@ package org.apache.hadoop.hbase.wal;
import java.io.Closeable;
import java.io.IOException;
import java.net.ConnectException;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
@@ -57,31 +57,31 @@ public abstract class AbstractWALRoller extends Thread
protected static final String WAL_ROLL_PERIOD_KEY =
"hbase.regionserver.logroll.period";
- protected final ConcurrentMap walNeedsRoll = new
ConcurrentHashMap<>();
+ protected final ConcurrentMap wals = new
ConcurrentHashMap<>();
protected final T abortable;
- private volatile long lastRollTime = System.currentTimeMillis();
// Period to roll log.
private final long rollPeriod;
private final int threadWakeFrequency;
// The interval to check low replication on hlog's pipeline
- private long checkLowReplicationInterval;
+ private final long checkLowReplicationInterval;
private volatile boolean running = true;
public void addWAL(WAL wal) {
// check without lock first
-if (walNeedsRoll.containsKey(wal)) {
+if (wals.containsKey(wal)) {
return;
}
// this is to avoid race between addWAL and requestRollAll.
synchronized (this) {
- if (walNeedsRoll.putIfAbsent(wal, Boolean.FALSE) == null) {
+ if (wals.putIfAbsent(wal, new RollController(wal)) == null) {
wal.registerWALActionsListener(new WALActionsListener() {
@Override
public void logRollRequested(WALActionsListener.RollRequestReason
reason) {
// TODO logs will contend with each other here, replace with e.g.
DelayedQueue
synchronized (AbstractWALRoller.this) {
- walNeedsRoll.put(wal, Boolean.TRUE);
+ RollController controller = wals.computeIfAbsent(wal, rc -> new
RollController(wal));
+ controller.requestRoll();
AbstractWALRoller.this.notifyAll();
}
}
@@ -92,9 +92,8 @@ public abstract class AbstractWALRoller
extends Thread
public void requestRollAll() {
synchronized (this) {
- List wals = new ArrayList(walNeedsRoll.keySet());
- for (WAL wal : wals) {
-walNeedsRoll.put(wal, Boolean.TRUE);
+ for (RollController controller : wals.values()) {
+controller.requestRoll();
}
notifyAll();
}
@@ -114,9 +113,9 @@ public abstract class AbstractWALRoller extends Thread
*/
private void checkLowReplication(long now) {
try {
- for (Entry entry : walNeedsRoll.entrySet()) {
+ for (Entry entry : wals.entrySet()) {
WAL wal = entry.getKey();
-boolean needRollAlready = entry.getValue();
+boolean needRollAlready = entry.getValue().needsRoll(now);
if (needRollAlready || !(wal instanceof AbstractFSWAL)) {
continue;
}
@@ -132,7
[hbase] branch master updated: HBASE-24758 Avoid flooding replication source RSes logs when no sinks… (#2118)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8c0d7fa HBASE-24758 Avoid flooding replication source RSes logs when
no sinks… (#2118)
8c0d7fa is described below
commit 8c0d7fa5b8971de8bc7062675ba96a6091263776
Author: Wellington Ramos Chevreuil
AuthorDate: Mon Jul 27 10:08:13 2020 +0100
HBASE-24758 Avoid flooding replication source RSes logs when no sinks…
(#2118)
Signed-off-by: Josh Elser
Signed-off-by: Viraj Jasani
---
.../hadoop/hbase/replication/HBaseReplicationEndpoint.java | 4 ++--
.../regionserver/HBaseInterClusterReplicationEndpoint.java | 12 ++--
.../replication/regionserver/ReplicationSinkManager.java | 3 +++
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
index 1ca70ad..3cde0d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java
@@ -168,8 +168,8 @@ public abstract class HBaseReplicationEndpoint extends
BaseReplicationEndpoint
}
/**
- * Get a list of all the addresses of all the region servers
- * for this peer cluster
+ * Get a list of all the addresses of all the available region servers
+ * for this peer cluster, or an empty list if no region servers available at
peer cluster.
* @return list of addresses
*/
// Synchronize peer cluster connection attempts to avoid races and rate
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index bfd689f..6a407e2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -127,6 +127,8 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
private boolean dropOnDeletedTables;
private boolean dropOnDeletedColumnFamilies;
private boolean isSerial = false;
+ //Initialising as 0 to guarantee at least one logging message
+ private long lastSinkFetchTime = 0;
/*
* Some implementations of HBaseInterClusterReplicationEndpoint may require
instantiate different
@@ -513,8 +515,14 @@ public class HBaseInterClusterReplicationEndpoint extends
HBaseReplicationEndpoi
int numSinks = replicationSinkMgr.getNumSinks();
if (numSinks == 0) {
- LOG.warn("{} No replication sinks found, returning without replicating. "
-+ "The source should retry with the same set of edits.", logPeerId());
+ if((System.currentTimeMillis() - lastSinkFetchTime) >=
(maxRetriesMultiplier*1000)) {
+LOG.warn(
+ "No replication sinks found, returning without replicating. "
++ "The source should retry with the same set of edits. Not logging
this again for "
++ "the next {} seconds.", maxRetriesMultiplier);
+lastSinkFetchTime = System.currentTimeMillis();
+ }
+ sleepForRetries("No sinks available at peer", sleepMultiplier);
return false;
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
index 21b07ac..db12dc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
@@ -150,6 +150,9 @@ public class ReplicationSinkManager {
*/
public synchronized void chooseSinks() {
List slaveAddresses = endpoint.getRegionServers();
+if(slaveAddresses.isEmpty()){
+ LOG.warn("No sinks available at peer. Will not be able to replicate");
+}
Collections.shuffle(slaveAddresses, ThreadLocalRandom.current());
int numSinks = (int) Math.ceil(slaveAddresses.size() * ratio);
sinks = slaveAddresses.subList(0, numSinks);
