[hbase] branch branch-2.4 updated: HBASE-25518 Support separate child regions to different region servers (#3001)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new df6e5dd HBASE-25518 Support separate child regions to different region servers (#3001) df6e5dd is described below commit df6e5dd0b1f3b7386735d97010ee9d89fcc5349b Author: haxiaolin AuthorDate: Fri Mar 19 03:38:17 2021 +0800 HBASE-25518 Support separate child regions to different region servers (#3001) Signed-off-by: stack --- .../java/org/apache/hadoop/hbase/HConstants.java | 15 + .../master/assignment/AssignmentManagerUtil.java | 63 + .../assignment/SplitTableRegionProcedure.java | 2 +- .../master/assignment/AssignmentTestingUtil.java | 25 + .../hbase/master/assignment/TestRegionSplit.java | 41 ++ ...ava => TestRegionSplitAndSeparateChildren.java} | 65 +++--- .../assignment/TestSplitTableRegionProcedure.java | 64 - 7 files changed, 176 insertions(+), 99 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 6853c61..93d288a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -157,6 +157,21 @@ public final class HConstants { /** Default value for the balancer period */ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 30; + /** + * Config key for enable/disable automatically separate child regions to different region servers + * in the procedure of split regions. One child will be kept to the server where parent + * region is on, and the other child will be assigned to a random server. + * See HBASE-25518. + */ + public static final String HBASE_ENABLE_SEPARATE_CHILD_REGIONS = +"hbase.master.auto.separate.child.regions.after.split.enabled"; + + /** + * Default value for automatically separate child regions to different region servers + * (set to "false" to keep all child regions to the server where parent region is on) + */ + public static final boolean DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS = false; + /** The name of the ensemble table */ public static final TableName ENSEMBLE_TABLE_NAME = TableName.valueOf("hbase:ensemble"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java index d401141..e7be28b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java @@ -27,6 +27,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS; /** * Utility for this assignment package only. @@ -199,6 +201,67 @@ final class AssignmentManagerUtil { return ArrayUtils.addAll(primaryRegionProcs, replicaRegionAssignProcs); } + /** + * Create round robin assign procedures for the given regions, + * according to the {@code regionReplication}. + * + * For rolling back, we will submit procedures directly to the {@code ProcedureExecutor}, so it is + * possible that we persist the newly scheduled procedures, and then crash before persisting the + * rollback state, so when we arrive here the second time, it is possible that some regions have + * already been associated with a TRSP. + * @param ignoreIfInTransition if true, will skip creating TRSP for the given region if it is + * already in transition, otherwise we will add an assert that it should not in + * transition. + */ + private static TransitRegionStateProcedure[] createRoundRobinAssignProcedures( +MasterProcedureEnv env, List regions, int regionReplication, +List serversToExclude, boolean ignoreIfInTransition) { +List regionsAndReplicas = new ArrayList<>(regions); +if (regionReplication != DEFAULT_REGI
[hbase] branch branch-2 updated: HBASE-25518 Support separate child regions to different region servers (#3001)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 566b051 HBASE-25518 Support separate child regions to different region servers (#3001) 566b051 is described below commit 566b0512a7faea09070d582853b221d1582004bc Author: haxiaolin AuthorDate: Fri Mar 19 03:38:17 2021 +0800 HBASE-25518 Support separate child regions to different region servers (#3001) Signed-off-by: stack --- .../java/org/apache/hadoop/hbase/HConstants.java | 15 + .../master/assignment/AssignmentManagerUtil.java | 63 + .../assignment/SplitTableRegionProcedure.java | 2 +- .../master/assignment/AssignmentTestingUtil.java | 25 + .../hbase/master/assignment/TestRegionSplit.java | 41 ++ ...ava => TestRegionSplitAndSeparateChildren.java} | 65 +++--- .../assignment/TestSplitTableRegionProcedure.java | 64 - 7 files changed, 176 insertions(+), 99 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 0165778..670cf49 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -157,6 +157,21 @@ public final class HConstants { /** Default value for the balancer period */ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 30; + /** + * Config key for enable/disable automatically separate child regions to different region servers + * in the procedure of split regions. One child will be kept to the server where parent + * region is on, and the other child will be assigned to a random server. + * See HBASE-25518. + */ + public static final String HBASE_ENABLE_SEPARATE_CHILD_REGIONS = +"hbase.master.auto.separate.child.regions.after.split.enabled"; + + /** + * Default value for automatically separate child regions to different region servers + * (set to "false" to keep all child regions to the server where parent region is on) + */ + public static final boolean DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS = false; + /** The name of the ensemble table */ public static final TableName ENSEMBLE_TABLE_NAME = TableName.valueOf("hbase:ensemble"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java index d401141..e7be28b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java @@ -27,6 +27,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS; /** * Utility for this assignment package only. @@ -199,6 +201,67 @@ final class AssignmentManagerUtil { return ArrayUtils.addAll(primaryRegionProcs, replicaRegionAssignProcs); } + /** + * Create round robin assign procedures for the given regions, + * according to the {@code regionReplication}. + * + * For rolling back, we will submit procedures directly to the {@code ProcedureExecutor}, so it is + * possible that we persist the newly scheduled procedures, and then crash before persisting the + * rollback state, so when we arrive here the second time, it is possible that some regions have + * already been associated with a TRSP. + * @param ignoreIfInTransition if true, will skip creating TRSP for the given region if it is + * already in transition, otherwise we will add an assert that it should not in + * transition. + */ + private static TransitRegionStateProcedure[] createRoundRobinAssignProcedures( +MasterProcedureEnv env, List regions, int regionReplication, +List serversToExclude, boolean ignoreIfInTransition) { +List regionsAndReplicas = new ArrayList<>(regions); +if (regionReplication != DEFAULT_REGION_R
[hbase] branch master updated: HBASE-25518 Support separate child regions to different region servers (#3001)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 585aca1 HBASE-25518 Support separate child regions to different region servers (#3001) 585aca1 is described below commit 585aca1f0579179ca234c0324420d27630054085 Author: haxiaolin AuthorDate: Fri Mar 19 03:38:17 2021 +0800 HBASE-25518 Support separate child regions to different region servers (#3001) Signed-off-by: stack --- .../java/org/apache/hadoop/hbase/HConstants.java | 15 + .../master/assignment/AssignmentManagerUtil.java | 63 + .../assignment/SplitTableRegionProcedure.java | 2 +- .../master/assignment/AssignmentTestingUtil.java | 25 + .../hbase/master/assignment/TestRegionSplit.java | 41 ++ ...ava => TestRegionSplitAndSeparateChildren.java} | 65 +++--- .../assignment/TestSplitTableRegionProcedure.java | 64 - 7 files changed, 176 insertions(+), 99 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 48fa00c..5796458 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -156,6 +156,21 @@ public final class HConstants { /** Default value for the balancer period */ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 30; + /** + * Config key for enable/disable automatically separate child regions to different region servers + * in the procedure of split regions. One child will be kept to the server where parent + * region is on, and the other child will be assigned to a random server. + * See HBASE-25518. + */ + public static final String HBASE_ENABLE_SEPARATE_CHILD_REGIONS = +"hbase.master.auto.separate.child.regions.after.split.enabled"; + + /** + * Default value for automatically separate child regions to different region servers + * (set to "false" to keep all child regions to the server where parent region is on) + */ + public static final boolean DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS = false; + /** The name of the ensemble table */ public static final TableName ENSEMBLE_TABLE_NAME = TableName.valueOf("hbase:ensemble"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java index 8666743..5f8f784 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java @@ -27,6 +27,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin; import org.apache.hadoop.hbase.client.RegionInfo; @@ -40,6 +41,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_ENABLE_SEPARATE_CHILD_REGIONS; /** * Utility for this assignment package only. @@ -189,6 +191,67 @@ final class AssignmentManagerUtil { return ArrayUtils.addAll(primaryRegionProcs, replicaRegionAssignProcs); } + /** + * Create round robin assign procedures for the given regions, + * according to the {@code regionReplication}. + * + * For rolling back, we will submit procedures directly to the {@code ProcedureExecutor}, so it is + * possible that we persist the newly scheduled procedures, and then crash before persisting the + * rollback state, so when we arrive here the second time, it is possible that some regions have + * already been associated with a TRSP. + * @param ignoreIfInTransition if true, will skip creating TRSP for the given region if it is + * already in transition, otherwise we will add an assert that it should not in + * transition. + */ + private static TransitRegionStateProcedure[] createRoundRobinAssignProcedures( +MasterProcedureEnv env, List regions, int regionReplication, +List serversToExclude, boolean ignoreIfInTransition) { +List regionsAndReplicas = new ArrayList<>(regions); +if (regionReplication != DEFAULT_REGION_REPLICA) { + + // collect th
[hbase] branch branch-1 updated (9534a1b -> 2a979e8)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 9534a1b HBASE-24395 Backport HBASE-20589 to branch-1 (#2690) add 2a979e8 HBASE-25622 Result#compareResults should compare tags. (#3004) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hbase/client/Result.java | 3 +- .../java/org/apache/hadoop/hbase/CellUtil.java | 8 .../org/apache/hadoop/hbase/client/TestResult.java | 47 ++ 3 files changed, 57 insertions(+), 1 deletion(-)
[hbase] branch branch-2 updated: HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 19d094b HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049) 19d094b is described below commit 19d094b96ffc322bc5b404f974404e6488b066e9 Author: bsglz <18031...@qq.com> AuthorDate: Fri Mar 19 03:13:06 2021 +0800 HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049) Signed-off-by: AnoopSamJohn Signed-off-by: stack --- .../apache/hadoop/hbase/regionserver/HRegion.java | 4 + .../hadoop/hbase/regionserver/MemStoreFlusher.java | 67 +-- .../hbase/regionserver/TestMemStoreFlusher.java| 98 ++ 3 files changed, 145 insertions(+), 24 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 09cc648..5bfeff7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -8380,6 +8380,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi flushesQueued.increment(); } + protected void decrementFlushesQueuedCount() { +flushesQueued.decrement(); + } + /** * If a handler thread is eligible for interrupt, make it ineligible. Should be paired * with {{@link #enableInterrupts()}. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 1f6a350..a35a0f1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -72,7 +72,7 @@ class MemStoreFlusher implements FlushRequester { // These two data members go together. Any entry in the one must have // a corresponding entry in the other. private final BlockingQueue flushQueue = new DelayQueue<>(); - private final Map regionsInQueue = new HashMap<>(); + protected final Map regionsInQueue = new HashMap<>(); private AtomicBoolean wakeupPending = new AtomicBoolean(); private final long threadWakeFrequency; @@ -126,20 +126,22 @@ class MemStoreFlusher implements FlushRequester { this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime", 9); int handlerCount = conf.getInt("hbase.hstore.flusher.count", 2); -if (handlerCount < 1) { - LOG.warn("hbase.hstore.flusher.count was configed to {} which is less than 1, corrected to 1", - handlerCount); - handlerCount = 1; +if (server != null) { + if (handlerCount < 1) { +LOG.warn("hbase.hstore.flusher.count was configed to {} which is less than 1, " ++ "corrected to 1", handlerCount); +handlerCount = 1; + } + LOG.info("globalMemStoreLimit=" + + TraditionalBinaryPrefix + .long2String(this.server.getRegionServerAccounting().getGlobalMemStoreLimit(), "", 1) + + ", globalMemStoreLimitLowMark=" + + TraditionalBinaryPrefix.long2String( + this.server.getRegionServerAccounting().getGlobalMemStoreLimitLowMark(), "", 1) + + ", Offheap=" + + (this.server.getRegionServerAccounting().isOffheap())); } this.flushHandlers = new FlushHandler[handlerCount]; -LOG.info("globalMemStoreLimit=" -+ TraditionalBinaryPrefix - .long2String(this.server.getRegionServerAccounting().getGlobalMemStoreLimit(), "", 1) -+ ", globalMemStoreLimitLowMark=" -+ TraditionalBinaryPrefix.long2String( - this.server.getRegionServerAccounting().getGlobalMemStoreLimitLowMark(), "", 1) -+ ", Offheap=" -+ (this.server.getRegionServerAccounting().isOffheap())); } public LongAdder getUpdatesBlockedMsHighWater() { @@ -462,18 +464,28 @@ class MemStoreFlusher implements FlushRequester { public boolean requestFlush(HRegion r, List families, FlushLifeCycleTracker tracker) { synchronized (regionsInQueue) { - if (!regionsInQueue.containsKey(r)) { -// This entry has no delay so it will be added at the top of the flush -// queue. It'll come out near immediately. -FlushRegionEntry fqe = new FlushRegionEntry(r, families, tracker); -this.regionsInQueue.put(r, fqe); -this.flushQueue.add(fqe); -r.incrementFlushesQueuedCount(); -return true; - } else { -tracker.notExecuted("Flush already requested on " + r); -return false; + FlushRegionEntry existFqe = regionsInQueue.get(r); + if (existFqe != null) { +
[hbase] branch master updated: HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new d93035a HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049) d93035a is described below commit d93035a1314ca8e0d838db7ec1cf525181e7076b Author: bsglz <18031...@qq.com> AuthorDate: Fri Mar 19 03:13:06 2021 +0800 HBASE-25643 The delayed FlushRegionEntry should be removed when we ne… (#3049) Signed-off-by: AnoopSamJohn Signed-off-by: stack --- .../apache/hadoop/hbase/regionserver/HRegion.java | 4 + .../hadoop/hbase/regionserver/MemStoreFlusher.java | 67 +-- .../hbase/regionserver/TestMemStoreFlusher.java| 98 ++ 3 files changed, 145 insertions(+), 24 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index a07bb15..4faecde 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -8417,6 +8417,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi flushesQueued.increment(); } + protected void decrementFlushesQueuedCount() { +flushesQueued.decrement(); + } + /** * If a handler thread is eligible for interrupt, make it ineligible. Should be paired * with {{@link #enableInterrupts()}. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 0a5ec91..61d5495 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -73,7 +73,7 @@ class MemStoreFlusher implements FlushRequester { // These two data members go together. Any entry in the one must have // a corresponding entry in the other. private final BlockingQueue flushQueue = new DelayQueue<>(); - private final Map regionsInQueue = new HashMap<>(); + protected final Map regionsInQueue = new HashMap<>(); private AtomicBoolean wakeupPending = new AtomicBoolean(); private final long threadWakeFrequency; @@ -127,20 +127,22 @@ class MemStoreFlusher implements FlushRequester { this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime", 9); int handlerCount = conf.getInt("hbase.hstore.flusher.count", 2); -if (handlerCount < 1) { - LOG.warn("hbase.hstore.flusher.count was configed to {} which is less than 1, corrected to 1", - handlerCount); - handlerCount = 1; +if (server != null) { + if (handlerCount < 1) { +LOG.warn("hbase.hstore.flusher.count was configed to {} which is less than 1, " ++ "corrected to 1", handlerCount); +handlerCount = 1; + } + LOG.info("globalMemStoreLimit=" + + TraditionalBinaryPrefix + .long2String(this.server.getRegionServerAccounting().getGlobalMemStoreLimit(), "", 1) + + ", globalMemStoreLimitLowMark=" + + TraditionalBinaryPrefix.long2String( + this.server.getRegionServerAccounting().getGlobalMemStoreLimitLowMark(), "", 1) + + ", Offheap=" + + (this.server.getRegionServerAccounting().isOffheap())); } this.flushHandlers = new FlushHandler[handlerCount]; -LOG.info("globalMemStoreLimit=" -+ TraditionalBinaryPrefix - .long2String(this.server.getRegionServerAccounting().getGlobalMemStoreLimit(), "", 1) -+ ", globalMemStoreLimitLowMark=" -+ TraditionalBinaryPrefix.long2String( - this.server.getRegionServerAccounting().getGlobalMemStoreLimitLowMark(), "", 1) -+ ", Offheap=" -+ (this.server.getRegionServerAccounting().isOffheap())); } public LongAdder getUpdatesBlockedMsHighWater() { @@ -467,18 +469,28 @@ class MemStoreFlusher implements FlushRequester { public boolean requestFlush(HRegion r, List families, FlushLifeCycleTracker tracker) { synchronized (regionsInQueue) { - if (!regionsInQueue.containsKey(r)) { -// This entry has no delay so it will be added at the top of the flush -// queue. It'll come out near immediately. -FlushRegionEntry fqe = new FlushRegionEntry(r, families, tracker); -this.regionsInQueue.put(r, fqe); -this.flushQueue.add(fqe); -r.incrementFlushesQueuedCount(); -return true; - } else { -tracker.notExecuted("Flush already requested on " + r); -return false; + FlushRegionEntry existFqe = regionsInQueue.get(r); + if (existFqe != null) { +
[hbase] branch branch-2.3 updated: Revert "HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)"
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 3e806a7 Revert "HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)" 3e806a7 is described below commit 3e806a7dcd362e8e0e9195072c7c6685aa3b5638 Author: stack AuthorDate: Thu Mar 18 12:09:16 2021 -0700 Revert "HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)" This reverts commit 728d4f5ab12fd2631b1ef0a7c61203e9acfb05f0. Reverted because it an addendum and main patch did not land on this branch so removing. --- bin/graceful_stop.sh | 6 +- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh index e565929..89e3dd9 100755 --- a/bin/graceful_stop.sh +++ b/bin/graceful_stop.sh @@ -32,7 +32,7 @@ moving regions" echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\ exit with error. Default value is INT_MAX." - echo " hostname Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh" + echo " hostname Hostname of server we are to stop" echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" echo " nob| nobalancer Do not manage balancer states. This is only used as optimization in \ rolling_restart.sh to avoid multiple calls to hbase shell" @@ -100,10 +100,6 @@ localhostname=`/bin/hostname` if [ "$localhostname" == "$hostname" ]; then local=true fi -if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then - local=true - hostname=$localhostname -fi if [ "$nob" == "true" ]; then log "[ $0 ] skipping disabling balancer -nob argument is used"
[hbase] branch branch-2.3 updated: HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 728d4f5 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) 728d4f5 is described below commit 728d4f5ab12fd2631b1ef0a7c61203e9acfb05f0 Author: Javier Akira Luca de Tena AuthorDate: Fri Mar 19 04:04:54 2021 +0900 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) Co-authored-by: Javier --- bin/graceful_stop.sh | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh index 89e3dd9..e565929 100755 --- a/bin/graceful_stop.sh +++ b/bin/graceful_stop.sh @@ -32,7 +32,7 @@ moving regions" echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\ exit with error. Default value is INT_MAX." - echo " hostname Hostname of server we are to stop" + echo " hostname Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh" echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" echo " nob| nobalancer Do not manage balancer states. This is only used as optimization in \ rolling_restart.sh to avoid multiple calls to hbase shell" @@ -100,6 +100,10 @@ localhostname=`/bin/hostname` if [ "$localhostname" == "$hostname" ]; then local=true fi +if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then + local=true + hostname=$localhostname +fi if [ "$nob" == "true" ]; then log "[ $0 ] skipping disabling balancer -nob argument is used"
[hbase] branch branch-2.4 updated: HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 90a58fe HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) 90a58fe is described below commit 90a58feacc0b452a64b74db8995058f89117f69c Author: Javier Akira Luca de Tena AuthorDate: Fri Mar 19 04:04:54 2021 +0900 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) Co-authored-by: Javier --- bin/graceful_stop.sh | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh index 8944653..05919ce 100755 --- a/bin/graceful_stop.sh +++ b/bin/graceful_stop.sh @@ -33,7 +33,7 @@ moving regions" echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\ exit with error. Default value is INT_MAX." - echo " hostname Hostname of server we are to stop" + echo " hostname Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh" echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" echo " nob|nobalancer Do not manage balancer states. This is only used as optimization in \ rolling_restart.sh to avoid multiple calls to hbase shell" @@ -108,6 +108,10 @@ localhostname=`/bin/hostname -f` if [ "$localhostname" == "$hostname" ]; then local=true fi +if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then + local=true + hostname=$localhostname +fi if [ "$nob" == "true" ]; then log "[ $0 ] skipping disabling balancer -nob argument is used"
[hbase] branch branch-2 updated: HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 9e701bb HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) 9e701bb is described below commit 9e701bb1d356c701d6a6b209229ef2a403695615 Author: Javier Akira Luca de Tena AuthorDate: Fri Mar 19 04:04:54 2021 +0900 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) Co-authored-by: Javier --- bin/graceful_stop.sh | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh index 8944653..05919ce 100755 --- a/bin/graceful_stop.sh +++ b/bin/graceful_stop.sh @@ -33,7 +33,7 @@ moving regions" echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\ exit with error. Default value is INT_MAX." - echo " hostname Hostname of server we are to stop" + echo " hostname Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh" echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" echo " nob|nobalancer Do not manage balancer states. This is only used as optimization in \ rolling_restart.sh to avoid multiple calls to hbase shell" @@ -108,6 +108,10 @@ localhostname=`/bin/hostname -f` if [ "$localhostname" == "$hostname" ]; then local=true fi +if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then + local=true + hostname=$localhostname +fi if [ "$nob" == "true" ]; then log "[ $0 ] skipping disabling balancer -nob argument is used"
[hbase] branch master updated: HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 0cead10 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) 0cead10 is described below commit 0cead102575d2750dbf83d6cd0ef9e398532f7b0 Author: Javier Akira Luca de Tena AuthorDate: Fri Mar 19 04:04:54 2021 +0900 HBOPS-25594 Make easier to use graceful_stop on localhost mode (#3054) Co-authored-by: Javier --- bin/graceful_stop.sh | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh index 8944653..05919ce 100755 --- a/bin/graceful_stop.sh +++ b/bin/graceful_stop.sh @@ -33,7 +33,7 @@ moving regions" echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\ exit with error. Default value is INT_MAX." - echo " hostname Hostname of server we are to stop" + echo " hostname Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh" echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" echo " nob|nobalancer Do not manage balancer states. This is only used as optimization in \ rolling_restart.sh to avoid multiple calls to hbase shell" @@ -108,6 +108,10 @@ localhostname=`/bin/hostname -f` if [ "$localhostname" == "$hostname" ]; then local=true fi +if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then + local=true + hostname=$localhostname +fi if [ "$nob" == "true" ]; then log "[ $0 ] skipping disabling balancer -nob argument is used"
[hbase] branch branch-2.4 updated: HBASE-25671 Backport HBASE-25608 to branch-2 (#3058)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 63f1580 HBASE-25671 Backport HBASE-25608 to branch-2 (#3058) 63f1580 is described below commit 63f158001537adfe655796526fe85d2a27ab5e3d Author: bitterfox AuthorDate: Fri Mar 19 04:00:25 2021 +0900 HBASE-25671 Backport HBASE-25608 to branch-2 (#3058) Signed-off-by: stack --- .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 72 +++- .../hbase/mapreduce/TestHFileOutputFormat2.java| 192 + 2 files changed, 263 insertions(+), 1 deletion(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index d9ba4bd..02b5768 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -176,6 +176,13 @@ public class HFileOutputFormat2 static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY = "hbase.mapreduce.use.multi.table.hfileoutputformat"; + public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster.zookeeper.quorum"; + public static final String REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster.zookeeper." + HConstants.CLIENT_PORT_STR; + public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster." + HConstants.ZOOKEEPER_ZNODE_PARENT; + public static final String STORAGE_POLICY_PROPERTY = HStore.BLOCK_STORAGE_POLICY_KEY; public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; @@ -288,7 +295,8 @@ public class HFileOutputFormat2 String tableName = Bytes.toString(tableNameBytes); if (tableName != null) { - try (Connection connection = ConnectionFactory.createConnection(conf); + try (Connection connection = ConnectionFactory.createConnection( +createRemoteClusterConf(conf)); RegionLocator locator = connection.getRegionLocator(TableName.valueOf(tableName))) { loc = locator.getRegionLocation(rowKey); @@ -358,6 +366,22 @@ public class HFileOutputFormat2 wl.written = 0; } + private Configuration createRemoteClusterConf(Configuration conf) { +final Configuration newConf = new Configuration(conf); + +final String quorum = conf.get(REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY); +final String clientPort = conf.get(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY); +final String parent = conf.get(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY); + +if (quorum != null && clientPort != null && parent != null) { + newConf.set(HConstants.ZOOKEEPER_QUORUM, quorum); + newConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.parseInt(clientPort)); + newConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parent); +} + +return newConf; + } + /* * Create a new StoreFile.Writer. * @return A WriterLength, containing a new StoreFile.Writer. @@ -536,6 +560,7 @@ public class HFileOutputFormat2 * Sets the output key/value class to match HFileOutputFormat2's requirements * Sets the reducer up to perform the appropriate sorting (either KeyValueSortReducer or * PutSortReducer) + * Sets the HBase cluster key to load region locations for locality-sensitive * * The user should be sure to set the map output value class to either KeyValue or Put before * running this function. @@ -543,6 +568,7 @@ public class HFileOutputFormat2 public static void configureIncrementalLoad(Job job, Table table, RegionLocator regionLocator) throws IOException { configureIncrementalLoad(job, table.getDescriptor(), regionLocator); +configureRemoteCluster(job, table.getConfiguration()); } /** @@ -673,6 +699,50 @@ public class HFileOutputFormat2 } /** + * Configure HBase cluster key for remote cluster to load region location for locality-sensitive + * if it's enabled. + * It's not necessary to call this method explicitly when the cluster key for HBase cluster to be + * used to load region location is configured in the job configuration. + * Call this method when another HBase cluster key is configured in the job configuration. + * For example, you should call when you load data from HBase cluster A using + * {@link TableInputFormat} and generate hfiles for HBase cluster B. + * Otherwise, HFileOutputFormat2 fetch location from
[hbase] branch branch-2 updated: HBASE-25671 Backport HBASE-25608 to branch-2 (#3058)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 150e02e HBASE-25671 Backport HBASE-25608 to branch-2 (#3058) 150e02e is described below commit 150e02ee58cc25e49f4a911a83c1368436d1d5c3 Author: bitterfox AuthorDate: Fri Mar 19 04:00:25 2021 +0900 HBASE-25671 Backport HBASE-25608 to branch-2 (#3058) Signed-off-by: stack --- .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 72 +++- .../hbase/mapreduce/TestHFileOutputFormat2.java| 192 + 2 files changed, 263 insertions(+), 1 deletion(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index d9ba4bd..02b5768 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -176,6 +176,13 @@ public class HFileOutputFormat2 static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY = "hbase.mapreduce.use.multi.table.hfileoutputformat"; + public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster.zookeeper.quorum"; + public static final String REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster.zookeeper." + HConstants.CLIENT_PORT_STR; + public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY = +"hbase.hfileoutputformat.remote.cluster." + HConstants.ZOOKEEPER_ZNODE_PARENT; + public static final String STORAGE_POLICY_PROPERTY = HStore.BLOCK_STORAGE_POLICY_KEY; public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; @@ -288,7 +295,8 @@ public class HFileOutputFormat2 String tableName = Bytes.toString(tableNameBytes); if (tableName != null) { - try (Connection connection = ConnectionFactory.createConnection(conf); + try (Connection connection = ConnectionFactory.createConnection( +createRemoteClusterConf(conf)); RegionLocator locator = connection.getRegionLocator(TableName.valueOf(tableName))) { loc = locator.getRegionLocation(rowKey); @@ -358,6 +366,22 @@ public class HFileOutputFormat2 wl.written = 0; } + private Configuration createRemoteClusterConf(Configuration conf) { +final Configuration newConf = new Configuration(conf); + +final String quorum = conf.get(REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY); +final String clientPort = conf.get(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY); +final String parent = conf.get(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY); + +if (quorum != null && clientPort != null && parent != null) { + newConf.set(HConstants.ZOOKEEPER_QUORUM, quorum); + newConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.parseInt(clientPort)); + newConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parent); +} + +return newConf; + } + /* * Create a new StoreFile.Writer. * @return A WriterLength, containing a new StoreFile.Writer. @@ -536,6 +560,7 @@ public class HFileOutputFormat2 * Sets the output key/value class to match HFileOutputFormat2's requirements * Sets the reducer up to perform the appropriate sorting (either KeyValueSortReducer or * PutSortReducer) + * Sets the HBase cluster key to load region locations for locality-sensitive * * The user should be sure to set the map output value class to either KeyValue or Put before * running this function. @@ -543,6 +568,7 @@ public class HFileOutputFormat2 public static void configureIncrementalLoad(Job job, Table table, RegionLocator regionLocator) throws IOException { configureIncrementalLoad(job, table.getDescriptor(), regionLocator); +configureRemoteCluster(job, table.getConfiguration()); } /** @@ -673,6 +699,50 @@ public class HFileOutputFormat2 } /** + * Configure HBase cluster key for remote cluster to load region location for locality-sensitive + * if it's enabled. + * It's not necessary to call this method explicitly when the cluster key for HBase cluster to be + * used to load region location is configured in the job configuration. + * Call this method when another HBase cluster key is configured in the job configuration. + * For example, you should call when you load data from HBase cluster A using + * {@link TableInputFormat} and generate hfiles for HBase cluster B. + * Otherwise, HFileOutputFormat2 fetch location from clu
[hbase] branch branch-2.3 updated: HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 9515b22 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) 9515b22 is described below commit 9515b22f45bbb999677b4f6b0148f7f71fdd8486 Author: Catalin Luca AuthorDate: Thu Mar 18 20:56:04 2021 +0200 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) Co-authored-by: Catalin Luca Signed-off-by: stack --- .../src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java index 5882d0f..2ce3d7d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.util.HashKey; import org.apache.hadoop.hbase.util.JenkinsHash; import org.apache.hadoop.hbase.util.MD5Hash; import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.IOUtils; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; @@ -744,8 +745,7 @@ public interface RegionInfo extends Comparable { } //assumption: if Writable serialization, it should be longer than pblen. -int read = in.read(pbuf); -if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen); +IOUtils.readFully(in, pbuf, 0, pblen); if (ProtobufUtil.isPBMagicPrefix(pbuf)) { return ProtobufUtil.toRegionInfo(HBaseProtos.RegionInfo.parseDelimitedFrom(in)); } else {
[hbase] branch branch-2.4 updated: HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 5b91500 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) 5b91500 is described below commit 5b91500d1ba88e6ca638356f269f8daf6b16b4c3 Author: Catalin Luca AuthorDate: Thu Mar 18 20:56:04 2021 +0200 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) Co-authored-by: Catalin Luca Signed-off-by: stack --- .../src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java index 0f51fed..e105dbc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.util.HashKey; import org.apache.hadoop.hbase.util.JenkinsHash; import org.apache.hadoop.hbase.util.MD5Hash; import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.IOUtils; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; @@ -750,8 +751,7 @@ public interface RegionInfo extends Comparable { } //assumption: if Writable serialization, it should be longer than pblen. -int read = in.read(pbuf); -if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen); +IOUtils.readFully(in, pbuf, 0, pblen); if (ProtobufUtil.isPBMagicPrefix(pbuf)) { return ProtobufUtil.toRegionInfo(HBaseProtos.RegionInfo.parseDelimitedFrom(in)); } else {
[hbase] branch branch-2 updated: HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new c9b81c3 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) c9b81c3 is described below commit c9b81c3f82e15d52951e6358552d3da044ebf205 Author: Catalin Luca AuthorDate: Thu Mar 18 20:56:04 2021 +0200 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) Co-authored-by: Catalin Luca Signed-off-by: stack --- .../src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java index 0f51fed..e105dbc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.util.HashKey; import org.apache.hadoop.hbase.util.JenkinsHash; import org.apache.hadoop.hbase.util.MD5Hash; import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.IOUtils; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; @@ -750,8 +751,7 @@ public interface RegionInfo extends Comparable { } //assumption: if Writable serialization, it should be longer than pblen. -int read = in.read(pbuf); -if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen); +IOUtils.readFully(in, pbuf, 0, pblen); if (ProtobufUtil.isPBMagicPrefix(pbuf)) { return ProtobufUtil.toRegionInfo(HBaseProtos.RegionInfo.parseDelimitedFrom(in)); } else {
[hbase] branch master updated (7ac1c8b -> 3ebb978)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from 7ac1c8b HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) add 3ebb978 HBASE-25674 - RegionInfo.parseFrom(DataInputStream) sometimes fails to read the protobuf magic marker (#3062) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[hbase] branch master updated: HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 7ac1c8b HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) 7ac1c8b is described below commit 7ac1c8bbf8da31586e786819ab515236ce6dbbdc Author: Michael Stack AuthorDate: Thu Mar 18 11:33:45 2021 -0700 HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) Don't have every handler update regionserver metrics on each scan#nextRaw; instead, do a batch update just before Scan returns. Otherwise, all running handlers end up contending on metrics update. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Update of regionserver metrics counters moved out to caller where can be done as a batch update instead of per-next. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java Class doc to encourage batch updating metrics. Remove the single update as unused anymore. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java Count calls to nextRaw. Update regionserver count in finally block when scan is done rather than per nextRaw call. Move all metrics updates to finally. Signed-off-by: Reid Chan Signed-off-by: Baiqiang Zhao --- .../hbase/regionserver/MetricsRegionServer.java | 18 ++ .../hadoop/hbase/regionserver/RSRpcServices.java | 19 +-- .../hadoop/hbase/regionserver/RegionScannerImpl.java | 5 + 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 86b97a2..1d9fb2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -28,12 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; /** - * - * This class is for maintaining the various regionserver statistics - * and publishing them through the metrics interfaces. - * + * Maintains regionserver statistics and publishes them through the metrics interfaces. * This class has a number of metrics variables that are publicly accessible; - * these variables (objects) have methods to update their values. + * these variables (objects) have methods to update their values. Batch your updates rather than + * call on each instance else all threads will do nothing but contend trying to maintain metric + * counters! */ @InterfaceStability.Evolving @InterfaceAudience.Private @@ -52,7 +51,9 @@ public class MetricsRegionServer { private MetricRegistry metricRegistry; private Timer bulkLoadTimer; + // Incremented once for each call to Scan#nextRaw private Meter serverReadQueryMeter; + // Incremented per write. private Meter serverWriteQueryMeter; protected long slowMetricTime; protected static final int DEFAULT_SLOW_METRIC_TIME = 1000; // milliseconds @@ -286,13 +287,6 @@ public class MetricsRegionServer { this.serverReadQueryMeter.mark(count); } - public void updateReadQueryMeter(TableName tn) { -if (tableMetrics != null && tn != null) { - tableMetrics.updateTableReadQueryMeter(tn); -} -this.serverReadQueryMeter.mark(); - } - public void updateWriteQueryMeter(TableName tn, long count) { if (tableMetrics != null && tn != null) { tableMetrics.updateTableWriteQueryMeter(tn, count); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index d06778a..5b0b77d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -3361,10 +3361,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler, // arbitrary 32. TODO: keep record of general size of results being returned. List values = new ArrayList<>(32); region.startRegionOperation(Operation.SCAN); +long before = EnvironmentEdgeManager.currentTime(); +// Used to check if we've matched the row limit set on the Scan +int numOfCompleteRows = 0; +// Count of times we call nextRaw; can be > numOfCompleteRows. +int numOfNextRawCalls = 0; try { int numOfResults = 0; - int numOfCompleteRows = 0; - long before = EnvironmentEdgeManager.currentTi
[hbase] branch branch-2 updated (97e0273 -> 75e2e7c)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 97e0273 HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057) add 75e2e7c HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) No new revisions were added by this update. Summary of changes: .../hbase/regionserver/MetricsRegionServer.java | 18 ++ .../hadoop/hbase/regionserver/RSRpcServices.java | 19 +-- .../hadoop/hbase/regionserver/RegionScannerImpl.java | 5 + 3 files changed, 20 insertions(+), 22 deletions(-)
[hbase] branch branch-2.3 updated: HBASE-25639 meta replica state is not respected during active master switch (#3052)
This is an automated email from the ASF dual-hosted git repository. huaxiangsun pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 0a4ddd6 HBASE-25639 meta replica state is not respected during active master switch (#3052) 0a4ddd6 is described below commit 0a4ddd6c3bcf50ce7f3563e1699145d38cdf9de0 Author: huaxiangsun AuthorDate: Thu Mar 18 11:35:46 2021 -0700 HBASE-25639 meta replica state is not respected during active master switch (#3052) Signed-off-by: stack --- .../org/apache/hadoop/hbase/master/HMaster.java| 2 +- .../hadoop/hbase/master/MasterMetaBootstrap.java | 27 +++--- .../hbase/master/assignment/AssignmentManager.java | 36 +--- .../master/TestMasterFailoverWithMetaReplica.java | 102 + 4 files changed, 138 insertions(+), 29 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index b78f7f3..5f8b63d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1018,7 +1018,7 @@ public class HMaster extends HRegionServer implements MasterServices { RegionState rs = this.assignmentManager.getRegionStates(). getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO); LOG.info("hbase:meta {}", rs); -if (rs != null && rs.isOffline()) { +if ((rs == null) || (rs != null && rs.isOffline())) { Optional optProc = procedureExecutor.getProcedures().stream() .filter(p -> p instanceof InitMetaProcedure).map(o -> (InitMetaProcedure) o).findAny(); initMetaProc = optProc.orElseGet(() -> { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java index da8d228..800ae97 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -61,25 +60,25 @@ class MasterMetaBootstrap { throw new IllegalStateException("hbase:meta must be initialized first before we can " + "assign out its replicas"); } -ServerName metaServername = MetaTableLocator.getMetaRegionLocation(this.master.getZooKeeper()); + for (int i = 1; i < numReplicas; i++) { // Get current meta state for replica from zk. - RegionState metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper(), i); RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica( - RegionInfoBuilder.FIRST_META_REGIONINFO, i); - LOG.debug(hri.getRegionNameAsString() + " replica region state from zookeeper=" + metaState); - if (metaServername.equals(metaState.getServerName())) { -metaState = null; -LOG.info(hri.getRegionNameAsString() + - " old location is same as current hbase:meta location; setting location as null..."); - } +RegionInfoBuilder.FIRST_META_REGIONINFO, i); + + RegionState rs = assignmentManager.getRegionStates().getRegionState(hri); + LOG.debug(hri.getRegionNameAsString() + " replica region state from zookeeper=" + rs); + // These assigns run inline. All is blocked till they complete. Only interrupt is shutting // down hosting server which calls AM#stop. - if (metaState != null && metaState.getServerName() != null) { -// Try to retain old assignment. -assignmentManager.assignAsync(hri, metaState.getServerName()); - } else { + if (rs == null) { assignmentManager.assignAsync(hri); + } else if (rs != null && rs.isOffline()) { +if (rs.getServerName() != null) { + assignmentManager.assignAsync(hri, rs.getServerName()); +} else { + assignmentManager.assignAsync(hri); +} } } unassignExcessMetaReplica(numReplicas); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index edc7dee..880de2a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -45,6 +45,7 @@ import org.apa
[hbase] branch branch-2.4 updated: HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061)
This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 3331e83 HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) 3331e83 is described below commit 3331e8307ae0da8dc0779751aafd38f034837fb5 Author: Michael Stack AuthorDate: Thu Mar 18 11:33:45 2021 -0700 HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) Don't have every handler update regionserver metrics on each scan#nextRaw; instead, do a batch update just before Scan returns. Otherwise, all running handlers end up contending on metrics update. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Update of regionserver metrics counters moved out to caller where can be done as a batch update instead of per-next. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java Class doc to encourage batch updating metrics. Remove the single update as unused anymore. M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java Count calls to nextRaw. Update regionserver count in finally block when scan is done rather than per nextRaw call. Move all metrics updates to finally. Signed-off-by: Reid Chan Signed-off-by: Baiqiang Zhao --- .../org/apache/hadoop/hbase/regionserver/HRegion.java | 3 --- .../hbase/regionserver/MetricsRegionServer.java | 18 ++ .../hadoop/hbase/regionserver/RSRpcServices.java | 19 +-- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index d54dc79..0f3649b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7282,9 +7282,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi metricsRegion.updateReadRequestCount(); } } - if (rsServices != null && rsServices.getMetrics() != null) { - rsServices.getMetrics().updateReadQueryMeter(getRegionInfo().getTable()); - } // If the size limit was reached it means a partial Result is being returned. Returning a // partial Result means that we should not reset the filters; filters should only be reset in diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 00324f3..f593f39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -28,12 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; /** - * - * This class is for maintaining the various regionserver statistics - * and publishing them through the metrics interfaces. - * + * Maintains regionserver statistics and publishes them through the metrics interfaces. * This class has a number of metrics variables that are publicly accessible; - * these variables (objects) have methods to update their values. + * these variables (objects) have methods to update their values. Batch your updates rather than + * call on each instance else all threads will do nothing but contend trying to maintain metric + * counters! */ @InterfaceStability.Evolving @InterfaceAudience.Private @@ -52,7 +51,9 @@ public class MetricsRegionServer { private MetricRegistry metricRegistry; private Timer bulkLoadTimer; + // Incremented once for each call to Scan#nextRaw private Meter serverReadQueryMeter; + // Incremented per write. private Meter serverWriteQueryMeter; protected long slowMetricTime; protected static final int DEFAULT_SLOW_METRIC_TIME = 1000; // milliseconds @@ -272,13 +273,6 @@ public class MetricsRegionServer { this.serverReadQueryMeter.mark(count); } - public void updateReadQueryMeter(TableName tn) { -if (tableMetrics != null && tn != null) { - tableMetrics.updateTableReadQueryMeter(tn); -} -this.serverReadQueryMeter.mark(); - } - public void updateWriteQueryMeter(TableName tn, long count) { if (tableMetrics != null && tn != null) { tableMetrics.updateTableWriteQueryMeter(tn, count); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/ja
[hbase] branch branch-2.3 updated (bd250f57 -> ec35193)
This is an automated email from the ASF dual-hosted git repository. stack pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git. from bd250f57 HBASE-25668 TestCurrentHourProvider fails 100% in branch-2.3" add ec35193 HBASE-25677 Server+table counters on each scan #nextRaw invocation becomes a bottleneck when heavy load (#3061) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hbase/regionserver/HRegion.java | 3 --- .../hbase/regionserver/MetricsRegionServer.java | 18 ++ .../hadoop/hbase/regionserver/RSRpcServices.java | 19 +-- 3 files changed, 19 insertions(+), 21 deletions(-)
[hbase] annotated tag rel/2.4.2 created (now 83b5291)
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a change to annotated tag rel/2.4.2 in repository https://gitbox.apache.org/repos/asf/hbase.git. at 83b5291 (tag) tagging 3e98c51c512cbd5ef779ae6bcef178ce89c46e37 (commit) replaces rel/2.4.1 by Andrew Purtell on Thu Mar 18 10:45:18 2021 -0700 - Log - Apache HBase release 2.4.2 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEEUPHou3xnqxS9/AohhZd1TdU2XM0FAmBTka4ACgkQhZd1TdU2 XM31qhAAtSqnqRlDv5qDtyoVy6vOsWZVDrDpxlz/T9wgf/Spc3CGYZ/njLtF28w0 yNE/lsu3FCxtuF3XYIkT2TRFoyjT2+tWxwh/E39R9OM81KAuVc5yFK5GafYlTx/B FvqA0U1PvrjOFz/SlcKPcJ/xIyJhyBmiTKRihwyQ6Q57VfxCk62vkwJAXyHcmI0I vPlxruWgSF8OCKIFWC3zla2JlUpjjixQAH9Vc+xQtX+IYgRbw+zP54Sd2UuKzpIZ hh6K9RrB42JgFblYl+sVfE2bG1nn3axXBEnDweJ4zW06ffwJaSvkzmH8h1zLwpyD hxovvIq1LEYvvhVgFsJvLGG9F34rGn5SsFbzl0/gppiPaijBQpNSC3yaovFu455V TSQkjBl1eqHY5EMiHW//9u1REj00WTFR6T9SoIlIuhsWGZcJoFeb/Ew2Fb6z/8lD WdzavPaRGA73TjA7VAEgJo7xr81MnmuKfXJJs2boDFqM8u/6wy05oKdHKB7soI8V HoZiKrI7MIFqdXDLrm3ktXK822okwa98zZgBnlPM0R0v3CLgYyZVPv/0tY4qzusg cB4hRIYntv/B6KVphAE02hO4HShxqYIUVIt8J2yHf8+R6tcMqCUM62Vi85P9BW05 wIOsIlX/8yUc9bU6yfqyNj2qPwGyfMbXtZ3Xb5ny1CBk++WA2Ls= =D2oZ -END PGP SIGNATURE- --- No new revisions were added by this update.
[hbase] branch branch-2 updated: HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057)
This is an automated email from the ASF dual-hosted git repository. anoopsamjohn pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 97e0273 HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057) 97e0273 is described below commit 97e02732b4352a9c273480302348a63a34fcb83e Author: Anoop Sam John AuthorDate: Thu Mar 18 21:18:25 2021 +0530 HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057) Signed-off-by: Viraj Jasani Signed-off-by: Wellington Chevreuil --- .../main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java index 8460a3a..ed2f977 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java @@ -127,7 +127,7 @@ public class ZKLeaderManager extends ZKListener { ZKUtil.deleteNode(watcher, leaderZNode); leaderExists.set(false); } else { - LOG.info("Found existing leader with ID: {}", Bytes.toStringBinary(nodeId)); + LOG.info("Found existing leader with ID: {}", Bytes.toStringBinary(currentId)); leaderExists.set(true); } } catch (KeeperException ke) {
[hbase] branch branch-2.2 updated: HBASE-25656 Backport to branch-2.2: [HBASE-25548 Optionally allow sna… (#3055)
This is an automated email from the ASF dual-hosted git repository. wchevreuil pushed a commit to branch branch-2.2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.2 by this push: new 36a0f01 HBASE-25656 Backport to branch-2.2: [HBASE-25548 Optionally allow sna… (#3055) 36a0f01 is described below commit 36a0f01b5500b53d3d878025506e3217158c732c Author: Wellington Ramos Chevreuil AuthorDate: Thu Mar 18 15:50:18 2021 + HBASE-25656 Backport to branch-2.2: [HBASE-25548 Optionally allow sna… (#3055) Signed-off-by: Peter Somogyi Signed-off-by: Michael Stack --- .../java/org/apache/hadoop/hbase/client/Admin.java | 47 + .../hadoop/hbase/client/SnapshotDescription.java | 44 ++-- .../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 8 +- .../src/main/protobuf/Snapshot.proto | 1 + .../hbase/master/snapshot/SnapshotManager.java | 4 + .../hbase/master/snapshot/TakeSnapshotHandler.java | 9 +- .../hbase/client/TestSnapshotFromClient.java | 2 +- .../master/snapshot/TestTakeSnapshotHandler.java | 111 + hbase-shell/src/main/ruby/hbase/admin.rb | 8 +- .../src/main/ruby/shell/commands/snapshot.rb | 2 +- src/main/asciidoc/_chapters/ops_mgt.adoc | 12 +++ 11 files changed, 234 insertions(+), 14 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 1746dd5..1e01cd1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1983,6 +1983,53 @@ public interface Admin extends Abortable, Closeable { IllegalArgumentException; /** + * Create typed snapshot of the table. Snapshots are considered unique based on the name of the + * snapshot. Snapshots are taken sequentially even when requested concurrently, across + * all tables. Attempts to take a snapshot with the same name (even a different type or with + * different parameters) will fail with a {@link SnapshotCreationException} indicating the + * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See + * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}. + * Snapshot can live with ttl seconds. + * + * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other + * snapshots stored on the cluster + * @param tableName name of the table to snapshot + * @param type type of snapshot to take + * @param snapshotProps snapshot additional properties e.g. TTL + * @throws IOException we fail to reach the master + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + default void snapshot(String snapshotName, TableName tableName, SnapshotType type, +Map snapshotProps) throws IOException, +SnapshotCreationException, IllegalArgumentException { +snapshot(new SnapshotDescription(snapshotName, tableName, type, snapshotProps)); + } + + /** + * Create typed snapshot of the table. Snapshots are considered unique based on the name of the + * snapshot. Snapshots are taken sequentially even when requested concurrently, across + * all tables. Attempts to take a snapshot with the same name (even a different type or with + * different parameters) will fail with a {@link SnapshotCreationException} indicating the + * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See + * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}. + * Snapshot can live with ttl seconds. + * + * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other + * snapshots stored on the cluster + * @param tableName name of the table to snapshot + * @param snapshotProps snapshot additional properties e.g. TTL + * @throws IOException we fail to reach the master + * @throws SnapshotCreationException if snapshot creation failed + * @throws IllegalArgumentException if the snapshot request is formatted incorrectly + */ + default void snapshot(String snapshotName, TableName tableName, +Map snapshotProps) throws IOException, +SnapshotCreationException, IllegalArgumentException { +snapshot(new SnapshotDescription(snapshotName, tableName, SnapshotType.FLUSH, snapshotProps)); + } + + /** * Take a snapshot and wait for the server to complete that snapshot (blocking). Only a single * snapshot should be taken at a time for an instance of HBase, or results may be undefined (you * can tell m
[hbase] branch master updated: HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057)
This is an automated email from the ASF dual-hosted git repository. anoopsamjohn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new a698b1e HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057) a698b1e is described below commit a698b1ea536874287bb9edaec165bcd7c794e598 Author: Anoop Sam John AuthorDate: Thu Mar 18 21:18:25 2021 +0530 HBASE-25673 Wrong log regarding current active master at ZKLeaderManager#waitToBecomeLeader (#3057) Signed-off-by: Viraj Jasani Signed-off-by: Wellington Chevreuil --- .../main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java index 8460a3a..ed2f977 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java @@ -127,7 +127,7 @@ public class ZKLeaderManager extends ZKListener { ZKUtil.deleteNode(watcher, leaderZNode); leaderExists.set(false); } else { - LOG.info("Found existing leader with ID: {}", Bytes.toStringBinary(nodeId)); + LOG.info("Found existing leader with ID: {}", Bytes.toStringBinary(currentId)); leaderExists.set(true); } } catch (KeeperException ke) {