hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
Repository: hbase
Updated Branches:
refs/heads/master a34f129af -> 8c74d177f
HBASE-20082 Fix findbugs errors only on master which are introduced by
HBASE-19397
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c74d177
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c74d177
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c74d177
Branch: refs/heads/master
Commit: 8c74d177f68bbd5412cef96dc33f16ba33ff7875
Parents: a34f129
Author: zhangduo
Authored: Mon Feb 26 10:22:09 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:50:51 2018 +0800
--
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++
.../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 1253ef9..ba9bcdc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends
Procedure
private PeerOperationType type;
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"IS2_INCONSISTENT_SYNC",
+ justification = "Will never change after construction")
private ServerName targetServer;
private boolean dispatched;
http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 4e1b20d..ebb8042 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -403,7 +403,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
toStartup.add(replicationSource);
}
}
-for (ReplicationSourceInterface replicationSource : oldsources) {
+for (ReplicationSourceInterface replicationSource : toStartup) {
replicationSource.startup();
}
}
[08/50] [abbrv] hbase git commit: HBASE-20043 ITBLL fails against hadoop3
HBASE-20043 ITBLL fails against hadoop3
Fix MoveRandomRegionOfTableAction. It depended on old AM behavior.
Make it do explicit move as is required in AMv3; w/o it, it was just
closing region causing test to fail.
Fix pom so hadoop3 profile specifies a different netty3 version.
Bunch of logging format change that came of trying trying to read
the spew from this test.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b3ae58e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b3ae58e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b3ae58e
Branch: refs/heads/HBASE-19397-branch-2
Commit: 8b3ae58e187f3d731a779528f565c53e40dd7695
Parents: 2d5065b
Author: Michael Stack
Authored: Fri Feb 23 14:53:52 2018 -0800
Committer: Michael Stack
Committed: Sat Feb 24 17:29:24 2018 -0800
--
.../apache/hadoop/hbase/io/ByteBufferPool.java | 5 +--
.../hadoop/hbase/chaos/actions/Action.java | 32 +-
.../actions/MoveRandomRegionOfTableAction.java | 13
.../chaos/actions/MoveRegionsOfTableAction.java | 35
.../chaos/monkies/PolicyBasedChaosMonkey.java | 2 +-
.../hbase/chaos/policies/PeriodicPolicy.java| 6 ++--
.../policies/PeriodicRandomActionPolicy.java| 3 +-
.../procedure2/RemoteProcedureDispatcher.java | 4 +--
.../hadoop/hbase/io/hfile/CacheConfig.java | 2 +-
.../hadoop/hbase/io/hfile/LruBlockCache.java| 14
.../apache/hadoop/hbase/ipc/NettyRpcServer.java | 4 +--
.../apache/hadoop/hbase/ipc/RpcExecutor.java| 10 +++---
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../RegionServerProcedureManagerHost.java | 8 ++---
.../hbase/regionserver/CompactingMemStore.java | 5 +--
.../hbase/regionserver/CompactionPipeline.java | 2 +-
.../hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/regionserver/HRegionServer.java | 4 +--
.../hadoop/hbase/regionserver/HStore.java | 2 +-
.../hbase/regionserver/HeapMemoryManager.java | 10 +++---
.../hadoop/hbase/regionserver/Leases.java | 3 +-
.../MemStoreCompactionStrategy.java | 5 ++-
.../hbase/regionserver/MemStoreCompactor.java | 3 +-
.../hadoop/hbase/regionserver/Segment.java | 2 +-
.../compactions/CompactionConfiguration.java| 9 ++---
.../compactions/ExploringCompactionPolicy.java | 6 ++--
.../PressureAwareThroughputController.java | 2 +-
.../replication/regionserver/Replication.java | 2 +-
pom.xml | 9 +++--
29 files changed, 108 insertions(+), 98 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b3ae58e/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
index 6c009b3..caca20b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
@@ -80,8 +80,9 @@ public class ByteBufferPool {
this.maxPoolSize = maxPoolSize;
this.directByteBuffer = directByteBuffer;
// TODO can add initialPoolSize config also and make those many BBs ready
for use.
-LOG.info("Created ByteBufferPool with bufferSize : " + bufferSize + " and
maxPoolSize : "
-+ maxPoolSize);
+LOG.info("Created with bufferSize={} and maxPoolSize={}",
+org.apache.hadoop.util.StringUtils.byteDesc(bufferSize),
+org.apache.hadoop.util.StringUtils.byteDesc(maxPoolSize));
this.count = new AtomicInteger(0);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b3ae58e/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
--
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
index ae8cd1f..2b2c1b8 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
@@ -138,63 +138,63 @@ public class Action {
}
protected void killMaster(ServerName server) throws IOException {
-LOG.info("Killing master:" + server);
+LOG.info("Killing master " + server);
cluster.killMaster(server);
cluster.waitForMasterToStop(server, killMasterTimeout);
-LOG.info("Killed master server:" + server);
+LOG.info("Killed master " + server);
}
protected void startMaster(ServerName server) throws IOException {
-LOG.info("Starting master:" + s
[40/50] [abbrv] hbase git commit: HBASE-19719 Fix checkstyle issues
HBASE-19719 Fix checkstyle issues
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8b58c36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8b58c36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8b58c36
Branch: refs/heads/HBASE-19397-branch-2
Commit: f8b58c36df94ee195458cb55f4f5d58d8084
Parents: 03984c6
Author: zhangduo
Authored: Sat Jan 6 08:30:55 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../hbase/replication/ReplicationStorageFactory.java | 2 +-
.../master/assignment/RegionTransitionProcedure.java | 4 ++--
.../hbase/master/procedure/RSProcedureDispatcher.java | 13 ++---
.../master/ReplicationPeerConfigUpgrader.java | 8
4 files changed, 13 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 60d0749..462cfed 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -27,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* For now we only have zk based implementation.
*/
@InterfaceAudience.Private
-public class ReplicationStorageFactory {
+public final class ReplicationStorageFactory {
private ReplicationStorageFactory() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 1724a38..8277dbe 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -36,11 +36,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
/**
* Base class for the Assign and Unassign Procedure.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 57a4535..6c78914 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
@@ -36,6 +35,12 @@ import org.apache.hadoop.ipc.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
@@ -47,12 +52,6 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR
import
org.apache.hadoop.hbase.shaded.protobu
[06/50] [abbrv] hbase git commit: HBASE-19583 update RM list to remove EOM versions
HBASE-19583 update RM list to remove EOM versions Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/675f76b5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/675f76b5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/675f76b5 Branch: refs/heads/HBASE-19397-branch-2 Commit: 675f76b5a4ea5aca9ba659d4426ebfa6f6a21855 Parents: 62b8da9 Author: Sean Busbey Authored: Sat Feb 24 13:03:22 2018 -0600 Committer: Sean Busbey Committed: Sat Feb 24 16:50:30 2018 -0600 -- src/main/asciidoc/_chapters/developer.adoc | 3 --- 1 file changed, 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/675f76b5/src/main/asciidoc/_chapters/developer.adoc -- diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc index ef9ae25..6d959c2 100644 --- a/src/main/asciidoc/_chapters/developer.adoc +++ b/src/main/asciidoc/_chapters/developer.adoc @@ -1539,9 +1539,6 @@ NOTE: End-of-life releases are not included in this list. | Release | Release Manager -| 1.1 -| Nick Dimiduk - | 1.2 | Sean Busbey
[37/50] [abbrv] hbase git commit: HBASE-19697 Remove TestReplicationAdminUsingProcedure
HBASE-19697 Remove TestReplicationAdminUsingProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d448c013
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d448c013
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d448c013
Branch: refs/heads/HBASE-19397-branch-2
Commit: d448c01303e4627ece76c13ab91688fe334bd6ab
Parents: 85e85be
Author: zhangduo
Authored: Wed Jan 3 21:13:57 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:03 2018 +0800
--
.../TestReplicationAdminUsingProcedure.java | 225 ---
1 file changed, 225 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d448c013/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
deleted file mode 100644
index 1300376..000
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client.replication;
-
-import java.io.IOException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ MediumTests.class, ClientTests.class })
-public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
-
- private static final String PEER_ID = "2";
- private static final Logger LOG =
Logger.getLogger(TestReplicationAdminUsingProcedure.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-conf1.setInt("hbase.multihconnection.threads.max", 10);
-
-// Start the master & slave mini cluster.
-TestReplicationBase.setUpBeforeClass();
-
-// Remove the replication peer
-hbaseAdmin.removeReplicationPeer(PEER_ID);
- }
-
- private void loadData(int startRowKey, int endRowKey) throws IOException {
-for (int i = startRowKey; i < endRowKey; i++) {
- byte[] rowKey = Bytes.add(row, Bytes.toBytes(i));
- Put put = new Put(rowKey);
- put.addColumn(famName, null, Bytes.toBytes(i));
- htable1.put(put);
-}
- }
-
- private void waitForReplication(int expectedRows, int retries)
- throws IOException, InterruptedException {
-Scan scan;
-for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i == retries - 1) {
-throw new IOException("Waited too much time for normal batch
replication");
- }
- try (ResultScanner scanner = htable2.getScanner(scan)) {
-int count = 0;
-for (Result res : scanner) {
- count++;
-}
-if (count != expectedRows) {
- LOG.info("Only got " + count + " rows, expected rows: " +
expectedRows);
- Thread.sleep(SLEEP_TIME);
-} else {
- return;
-}
- }
-}
- }
-
- @Before
- public void setUp() throws IOException {
-ReplicationPeerConfig rpc = new ReplicationPeerConfig();
-
[20/50] [abbrv] hbase git commit: HBASE-19630 Add peer cluster key check when add new replication peer
HBASE-19630 Add peer cluster key check when add new replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3be7732
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3be7732
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3be7732
Branch: refs/heads/HBASE-19397-branch-2
Commit: b3be773239a141928066bbaabda4571a8689d146
Parents: 751a6eb
Author: Guanghao Zhang
Authored: Tue Dec 26 21:10:00 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../replication/ReplicationPeerManager.java | 54
.../replication/TestReplicationAdmin.java | 22
2 files changed, 54 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3be7732/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 84abfeb..b78cbce 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,6 +43,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -216,36 +218,36 @@ public final class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
- /**
- * If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
- * Then allow config exclude namespaces or exclude table-cfs which can't be
replicated to peer
- * cluster.
- *
- * If replicate_all flag is false, it means all user tables can't be
replicated to peer cluster.
- * Then allow to config namespaces or table-cfs which will be replicated to
peer cluster.
- */
- private static void checkPeerConfig(ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException {
+ private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws
DoNotRetryIOException {
+checkClusterKey(peerConfig.getClusterKey());
+
if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty()) ||
-(peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
-throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly " +
- "when you want replicate all cluster");
+ // If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
+ // Then allow config exclude namespaces or exclude table-cfs which can't
be replicated to peer
+ // cluster.
+ if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty())
+ || (peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
+throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly "
++ "when you want replicate all cluster");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
peerConfig.getExcludeTableCFsMap());
} else {
- if ((peerConfig.getExcludeNamespaces() != null &&
-!peerConfig.getExcludeNamespaces().isEmpty()) ||
-(peerConfig.getExcludeTableCFsMap() != null &&
- !peerConfig.getExcludeTableCFsMap().isEmpty())) {
+ // If replicate_all flag is false, it means all user tables can't be
replicated to peer
+ // cluster. Then allow to config namespaces or table-cfs which will be
replicated to peer
+ // cluster.
+ if ((peerConfig.getExcludeNamespaces() != null
+ && !peerConfig.getExcludeNamespaces().isEmpty())
+ || (peerConfig.getExcludeTableCFsMap() != null
+ && !peerConfig.getExcludeTableCFsMap().isEmpty())) {
throw new DoNotRetryIOException(
-"Need clean exclude-namespaces or exclude-table-cfs config
firstly"
[46/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
index 4c1b94f..6d2b578 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
@@ -28,12 +28,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -41,6 +40,8 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
@@ -71,9 +72,6 @@ public class TestNamespaceReplication extends
TestReplicationBase {
private static final byte[] val = Bytes.toBytes("myval");
- private static HTableDescriptor tabA;
- private static HTableDescriptor tabB;
-
private static Connection connection1;
private static Connection connection2;
private static Admin admin1;
@@ -93,23 +91,21 @@ public class TestNamespaceReplication extends
TestReplicationBase {
admin2.createNamespace(NamespaceDescriptor.create(ns1).build());
admin2.createNamespace(NamespaceDescriptor.create(ns2).build());
-tabA = new HTableDescriptor(tabAName);
-HColumnDescriptor fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
+TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(tabAName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabA = builder.build();
admin1.createTable(tabA);
admin2.createTable(tabA);
-tabB = new HTableDescriptor(tabBName);
-fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
+builder = TableDescriptorBuilder.newBuilder(tabBName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabB = builder.build();
admin1.createTable(tabB);
admin2.createTable(tabB);
}
@@ -137,22 +133,24 @@ public class TestNamespaceReplication extends
TestReplicationBase {
@Test
public void testNamespaceReplication() throws Exception {
+String peerId = "2";
+
Table htab1A = connection1.getTable(tabAName);
Table htab2A = connection2.getTable(tabAName);
Table htab1B = connection1.getTable(tabBName);
Table htab2B = connection2.getTable(tabBName);
-ReplicationPeerConfig rpc = admin.getPeerConfig("2");
-rpc.setReplicateAllUserTables(false);
-admin.updatePeerConfig("2", rpc);
+ReplicationPeerConfig rpc = admin1.getReplicationPeerConfig(peerId);
+admin1.updateReplicationPeerConfig(peerId,
+
ReplicationPeerConfig.newBuilder(rpc).setReplicateAllUserTables(false).build());
// add ns1 to peer config which replicate to cluster2
-rpc = admin.getPeerConfig("2");
+rpc = admin1.getReplicationPeerConfig(peerId);
Set namespaces = new HashSet<>();
namespaces.add(ns1);
-rpc.setNamespaces(namespaces);
-admin.updatePeerConfig("2", rpc);
+admin1.updateReplicationPeerConfig(peerId,
+ Replication
[15/50] [abbrv] hbase git commit: HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure
HBASE-19525 RS side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8eb68c40
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8eb68c40
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8eb68c40
Branch: refs/heads/HBASE-19397-branch-2
Commit: 8eb68c409531641ccf21215b1c7a11fa0b3f8bf7
Parents: 4a497cc
Author: huzheng
Authored: Wed Dec 20 10:47:18 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hadoop/hbase/protobuf/ProtobufUtil.java | 11 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 13 +-
.../hbase/replication/ReplicationListener.java | 14 --
.../hbase/replication/ReplicationPeer.java | 28 ++-
.../replication/ReplicationPeerZKImpl.java | 186 ---
.../replication/ReplicationPeersZKImpl.java | 19 +-
.../replication/ReplicationTrackerZKImpl.java | 73 +-
.../regionserver/ReplicationSourceService.java | 6 +
.../handler/RSProcedureHandler.java | 3 +
.../replication/BaseReplicationEndpoint.java| 2 +-
.../regionserver/PeerProcedureHandler.java | 38
.../regionserver/PeerProcedureHandlerImpl.java | 81 +++
.../regionserver/RefreshPeerCallable.java | 39 +++-
.../replication/regionserver/Replication.java | 9 +
.../regionserver/ReplicationSource.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 37 ++-
.../TestReplicationAdminUsingProcedure.java | 226 +++
.../replication/DummyModifyPeerProcedure.java | 48
.../TestDummyModifyPeerProcedure.java | 80 ---
.../TestReplicationTrackerZKImpl.java | 61 -
.../TestReplicationSourceManager.java | 32 ++-
21 files changed, 532 insertions(+), 482 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8eb68c40/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 3033da7..f11fcf6 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -191,7 +193,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMagic#PB_MAGIC}.length.
*/
public static byte [] prependPBMagic(final byte [] bytes) {
-return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
+return Bytes.add(PB_MAGIC, bytes);
}
/**
@@ -216,10 +218,11 @@ public final class ProtobufUtil {
* @param bytes bytes to check
* @throws DeserializationException if we are missing the pb magic prefix
*/
- public static void expectPBMagicPrefix(final byte [] bytes) throws
DeserializationException {
+ public static void expectPBMagicPrefix(final byte[] bytes) throws
DeserializationException {
if (!isPBMagicPrefix(bytes)) {
- throw new DeserializationException("Missing pb magic " +
- Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
+ String bytesPrefix = bytes == null ? "null" :
Bytes.toStringBinary(bytes, 0, PB_MAGIC.length);
+ throw new DeserializationException(
+ "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix, bytes: "
+ bytesPrefix);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8eb68c40/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 4472846..520a4cd 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.shaded.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -277,7 +279,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMag
[32/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6bbc6071
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6bbc6071
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6bbc6071
Branch: refs/heads/HBASE-19397-branch-2
Commit: 6bbc60719719579833ffd2a2905009f43a27bdd5
Parents: 296d586
Author: zhangduo
Authored: Wed Dec 27 22:03:51 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:24 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 9 +-
.../hbase/replication/ReplicationQueues.java| 161 ---
.../replication/ReplicationQueuesArguments.java | 70 ---
.../replication/ReplicationQueuesZKImpl.java| 417 -
.../hbase/replication/ReplicationTableBase.java | 442 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +-
.../replication/ZKReplicationQueueStorage.java | 22 +
.../replication/TestReplicationStateBasic.java | 131 +++---
.../replication/TestReplicationStateZKImpl.java | 41 +-
.../regionserver/DumpReplicationQueues.java | 15 +-
.../RecoveredReplicationSource.java | 17 +-
.../RecoveredReplicationSourceShipper.java | 22 +-
.../replication/regionserver/Replication.java | 20 +-
.../regionserver/ReplicationSource.java | 16 +-
.../ReplicationSourceInterface.java | 11 +-
.../regionserver/ReplicationSourceManager.java | 265 ++-
.../regionserver/ReplicationSyncUp.java | 32 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 12 +-
.../cleaner/TestReplicationHFileCleaner.java| 23 +-
.../cleaner/TestReplicationZKNodeCleaner.java | 22 +-
.../replication/ReplicationSourceDummy.java | 6 +-
.../replication/TestReplicationSyncUpTool.java | 4 +-
.../TestReplicationSourceManager.java | 97 ++--
.../TestReplicationSourceManagerZkImpl.java | 57 +--
24 files changed, 365 insertions(+), 1568 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c1c213..5e70e57 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.commons.lang3.reflect.ConstructorUtils;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A factory class for instantiating replication objects that deal with
replication state.
@@ -30,12 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationQueues
getReplicationQueues(ReplicationQueuesArguments args)
- throws Exception {
-return (ReplicationQueues)
ConstructorUtils.invokeConstructor(ReplicationQueuesZKImpl.class,
- args);
- }
-
public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
deleted file mode 100644
index a2d21f7..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apa
[12/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
HBASE-19543 Abstract a replication storage interface to extract the zk specific
code
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d8f09f1a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d8f09f1a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d8f09f1a
Branch: refs/heads/HBASE-19397-branch-2
Commit: d8f09f1aad7dda4cb3ffd1776edcb98c9de54d20
Parents: 8eb68c4
Author: zhangduo
Authored: Fri Dec 22 14:37:28 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hadoop/hbase/util/CollectionUtils.java | 3 +
hbase-replication/pom.xml | 12 +
.../replication/ReplicationPeerStorage.java | 74
.../replication/ReplicationQueueStorage.java| 164 +++
.../replication/ReplicationStateZKBase.java | 1 -
.../replication/ReplicationStorageFactory.java | 49 +++
.../replication/ZKReplicationPeerStorage.java | 164 +++
.../replication/ZKReplicationQueueStorage.java | 425 +++
.../replication/ZKReplicationStorageBase.java | 75
.../TestZKReplicationPeerStorage.java | 171
.../TestZKReplicationQueueStorage.java | 171
.../org/apache/hadoop/hbase/master/HMaster.java | 34 +-
.../hadoop/hbase/master/MasterServices.java | 6 +-
.../master/procedure/MasterProcedureEnv.java| 24 +-
.../master/replication/AddPeerProcedure.java| 6 +-
.../replication/DisablePeerProcedure.java | 7 +-
.../master/replication/EnablePeerProcedure.java | 6 +-
.../master/replication/ModifyPeerProcedure.java | 41 +-
.../master/replication/RemovePeerProcedure.java | 6 +-
.../master/replication/ReplicationManager.java | 199 -
.../replication/ReplicationPeerManager.java | 331 +++
.../replication/UpdatePeerConfigProcedure.java | 7 +-
.../replication/TestReplicationAdmin.java | 64 ++-
.../hbase/master/MockNoopMasterServices.java| 13 +-
.../hbase/master/TestMasterNoCluster.java | 3 +-
.../TestReplicationDisableInactivePeer.java | 6 +-
26 files changed, 1749 insertions(+), 313 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
index 875b124..8bbb6f1 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
@@ -107,6 +107,9 @@ public class CollectionUtils {
return list.get(list.size() - 1);
}
+ public static List nullToEmpty(List list) {
+return list != null ? list : Collections.emptyList();
+ }
/**
* In HBASE-16648 we found that ConcurrentHashMap.get is much faster than
computeIfAbsent if the
* value already exists. Notice that the implementation does not guarantee
that the supplier will
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-replication/pom.xml
--
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index edce309..77517cc 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -104,6 +104,18 @@
org.apache.hbase
hbase-zookeeper
+
+ org.apache.hbase
+ hbase-common
+ test-jar
+ test
+
+
+ org.apache.hbase
+ hbase-zookeeper
+ test-jar
+ test
+
org.apache.commons
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
new file mode 100644
index 000..e00cd0d
--- /dev/null
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *
[28/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 5b7bab8..ce9882a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -160,7 +160,6 @@ public class ReplicationSourceManager implements
ReplicationListener {
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationTracker.registerListener(this);
-this.replicationPeers.getAllPeerIds();
// It's preferable to failover 1 RS at a time, but with good zk servers
// more could be processed at the same time.
int nbWorkers = conf.getInt("replication.executor.workers", 1);
@@ -260,8 +259,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
}
List otherRegionServers =
replicationTracker.getListOfRegionServers().stream()
.map(ServerName::valueOf).collect(Collectors.toList());
-LOG.info(
- "Current list of replicators: " + currentReplicators + " other RSs: " +
otherRegionServers);
+LOG.info("Current list of replicators: " + currentReplicators + " other
RSs: "
++ otherRegionServers);
// Look if there's anything to process after a restart
for (ServerName rs : currentReplicators) {
@@ -278,7 +277,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
* The returned future is for adoptAbandonedQueues task.
*/
Future init() throws IOException, ReplicationException {
-for (String id : this.replicationPeers.getConnectedPeerIds()) {
+for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
if (replicationForBulkLoadDataEnabled) {
// Check if peer exists in hfile-refs queue, if not add it. This can
happen in the case
@@ -297,8 +296,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException,
ReplicationException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(id);
-ReplicationPeer peer = replicationPeers.getConnectedPeer(id);
+ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id);
+ReplicationPeer peer = replicationPeers.getPeer(id);
ReplicationSourceInterface src = getReplicationSource(id, peerConfig,
peer);
synchronized (this.walsById) {
this.sources.add(src);
@@ -344,7 +343,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void deleteSource(String peerId, boolean closeConnection) {
abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(),
peerId));
if (closeConnection) {
- this.replicationPeers.peerDisconnected(peerId);
+ this.replicationPeers.removePeer(peerId);
}
}
@@ -437,12 +436,12 @@ public class ReplicationSourceManager implements
ReplicationListener {
// update replication queues on ZK
// synchronize on replicationPeers to avoid adding source for the
to-be-removed peer
synchronized (replicationPeers) {
- for (String id : replicationPeers.getConnectedPeerIds()) {
+ for (String id : replicationPeers.getAllPeerIds()) {
try {
this.queueStorage.addWAL(server.getServerName(), id, logName);
} catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue" +
-" when creating a new source, queueId=" + id + ", filename=" +
logName, e);
+ throw new IOException("Cannot add log to replication queue"
+ + " when creating a new source, queueId=" + id + ", filename=" +
logName, e);
}
}
}
@@ -587,7 +586,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void addPeer(String id) throws ReplicationException, IOException {
LOG.info("Trying to add peer, peerId: " + id);
-boolean added = this.replicationPeers.peerConnected(id);
+boolean added = this.replicationPeers.addPeer(id);
if (added) {
LOG.info("Peer " + id + " connected success, trying to start the
replication source thread.");
addSource(id);
@@ -723,16 +722,21 @@ public class ReplicationSourceManager implements
ReplicationListener {
// there is not an actual peer defined corresponding to peerId for
the failover.
ReplicationQueueInfo replicationQueueInfo = new
ReplicationQueueInfo(peerId);
[50/50] [abbrv] hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
HBASE-20082 Fix findbugs errors only on master which are introduced by
HBASE-19397
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/732d2279
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/732d2279
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/732d2279
Branch: refs/heads/HBASE-19397-branch-2
Commit: 732d22793808627c1443b054bbc65ba923b37bc3
Parents: 2d5c4be
Author: zhangduo
Authored: Mon Feb 26 10:22:09 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:53 2018 +0800
--
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++
.../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/732d2279/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 1253ef9..ba9bcdc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends
Procedure
private PeerOperationType type;
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"IS2_INCONSISTENT_SYNC",
+ justification = "Will never change after construction")
private ServerName targetServer;
private boolean dispatched;
http://git-wip-us.apache.org/repos/asf/hbase/blob/732d2279/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 6965f55..d11dc8e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -402,7 +402,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
toStartup.add(replicationSource);
}
}
-for (ReplicationSourceInterface replicationSource : oldsources) {
+for (ReplicationSourceInterface replicationSource : toStartup) {
replicationSource.startup();
}
}
[13/50] [abbrv] hbase git commit: HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes
HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer
Procedure classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4a497cce
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4a497cce
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4a497cce
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4a497cced4ad485346e7fd49b7952e05d8be6c1f
Parents: de68d00
Author: zhangduo
Authored: Thu Dec 21 21:59:46 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hadoop/hbase/master/replication/AddPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/DisablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/EnablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/ModifyPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RemovePeerProcedure.java | 6 +++---
.../hbase/master/replication/UpdatePeerConfigProcedure.java| 6 +++---
7 files changed, 21 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index c3862d8..066c3e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -28,6 +26,8 @@ import
org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData;
@@ -37,7 +37,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.A
@InterfaceAudience.Private
public class AddPeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(AddPeerProcedure.class);
private ReplicationPeerConfig peerConfig;
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 0b32db9..9a28de6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for disabling a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DisablePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(DisablePeerProcedure.class);
public DisablePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
--
diff --git
a/hbase-server/src/mai
[48/50] [abbrv] hbase git commit: HBASE-19923 Reset peer state and config when refresh replication source failed
HBASE-19923 Reset peer state and config when refresh replication source failed
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d5c4be1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d5c4be1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d5c4be1
Branch: refs/heads/HBASE-19397-branch-2
Commit: 2d5c4be16fd214f3c118255daacdcd4f64e4476c
Parents: 1c4938d
Author: Guanghao Zhang
Authored: Tue Feb 6 14:58:39 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../hbase/replication/ReplicationPeerImpl.java | 4 ++--
.../regionserver/PeerProcedureHandlerImpl.java | 24
2 files changed, 22 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5c4be1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 604e0bb..d656466 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -54,11 +54,11 @@ public class ReplicationPeerImpl implements ReplicationPeer
{
this.peerConfigListeners = new ArrayList<>();
}
- void setPeerState(boolean enabled) {
+ public void setPeerState(boolean enabled) {
this.peerState = enabled ? PeerState.ENABLED : PeerState.DISABLED;
}
- void setPeerConfig(ReplicationPeerConfig peerConfig) {
+ public void setPeerConfig(ReplicationPeerConfig peerConfig) {
this.peerConfig = peerConfig;
peerConfigListeners.forEach(listener ->
listener.peerConfigUpdated(peerConfig));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5c4be1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index ce8fdae..a02d181 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -62,18 +62,26 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private void refreshPeerState(String peerId) throws ReplicationException,
IOException {
PeerState newState;
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+PeerState oldState = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- PeerState oldState = peer.getPeerState();
+ oldState = peer.getPeerState();
newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
// RS need to start work with the new replication state change
if (oldState.equals(PeerState.ENABLED) &&
newState.equals(PeerState.DISABLED)) {
replicationSourceManager.refreshSources(peerId);
}
+ success = true;
} finally {
+ if (!success && peer != null) {
+// Reset peer state if refresh source failed
+peer.setPeerState(oldState.equals(PeerState.ENABLED));
+ }
peerLock.unlock();
}
}
@@ -91,19 +99,27 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+ReplicationPeerConfig oldConfig = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- ReplicationPeerConfig oldConfig = peer.getPeerConfig();
+ oldConfig = peer.getPeerConfig();
ReplicationPeerConfig newConfig =
repl
[41/50] [abbrv] hbase git commit: HBASE-19936 Introduce a new base class for replication peer procedure
HBASE-19936 Introduce a new base class for replication peer procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1c4938da
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1c4938da
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1c4938da
Branch: refs/heads/HBASE-19397-branch-2
Commit: 1c4938dab8730eb596ecfa0ab414334887fd14a4
Parents: 0502464
Author: zhangduo
Authored: Mon Feb 5 16:14:25 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 2 +-
.../replication/AbstractPeerProcedure.java | 97
.../master/replication/ModifyPeerProcedure.java | 67 +-
3 files changed, 102 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c4938da/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index e45e12f..726217f 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -399,7 +399,7 @@ message RefreshPeerParameter {
required ServerName target_server = 3;
}
-message ModifyPeerStateData {
+message PeerProcedureStateData {
required string peer_id = 1;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c4938da/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
new file mode 100644
index 000..0ad8a63
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerProcedureStateData;
+
+/**
+ * The base class for all replication peer related procedure.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractPeerProcedure
+extends StateMachineProcedure implements
PeerProcedureInterface {
+
+ protected String peerId;
+
+ private volatile boolean locked;
+
+ // used to keep compatible with old client where we can only returns after
updateStorage.
+ protected ProcedurePrepareLatch latch;
+
+ protected AbstractPeerProcedure() {
+ }
+
+ protected AbstractPeerProcedure(String peerId) {
+this.peerId = peerId;
+this.latch = ProcedurePrepareLatch.createLatch(2, 0);
+ }
+
+ public ProcedurePrepareLatch getLatch() {
+return latch;
+ }
+
+ @Override
+ public String getPeerId() {
+return peerId;
+ }
+
+ @Override
+ protected LockState acquireLock(MasterProcedureEnv env) {
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
+ }
+
+ @Override
+ protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
+env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
+ }
+
+ @Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protecte
[09/50] [abbrv] hbase git commit: HBASE-20065 Addendum remove wrong comment
HBASE-20065 Addendum remove wrong comment
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10270e36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10270e36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10270e36
Branch: refs/heads/HBASE-19397-branch-2
Commit: 10270e36c9bd8571135ca7c8692a9e1ece0a6a45
Parents: 8b3ae58
Author: zhangduo
Authored: Mon Feb 26 09:48:41 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 09:48:55 2018 +0800
--
.../src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java| 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/10270e36/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 803f183..e779054 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -1535,7 +1535,6 @@ public class MetaTableAccessor {
RegionInfo regionA, RegionInfo regionB, ServerName sn, int
regionReplication)
throws IOException {
try (Table meta = getMetaHTable(connection)) {
- // use the maximum of what master passed us vs local time.
long time = EnvironmentEdgeManager.currentTime();
// Put for parent
[02/50] [abbrv] hbase git commit: HBASE-20048 Revert serial replication feature
HBASE-20048 Revert serial replication feature
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4ddfdaff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4ddfdaff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4ddfdaff
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4ddfdaffdcf83d50aac26fcff5e1ac4ecd575335
Parents: 8a22e41
Author: zhangduo
Authored: Fri Feb 23 08:51:37 2018 +0800
Committer: zhangduo
Committed: Fri Feb 23 13:54:10 2018 +0800
--
.../apache/hadoop/hbase/HTableDescriptor.java | 8 -
.../apache/hadoop/hbase/MetaTableAccessor.java | 296 ++
.../hadoop/hbase/client/TableDescriptor.java| 9 +-
.../hbase/client/TableDescriptorBuilder.java| 17 +-
.../client/replication/ReplicationAdmin.java| 10 +-
.../client/TestTableDescriptorBuilder.java | 25 +-
.../org/apache/hadoop/hbase/HConstants.java | 33 --
.../src/main/resources/hbase-default.xml| 13 -
hbase-protocol/src/main/protobuf/WAL.proto | 1 -
.../org/apache/hadoop/hbase/master/HMaster.java | 8 +-
.../master/assignment/RegionStateStore.java | 23 +-
.../master/cleaner/ReplicationMetaCleaner.java | 191 -
.../hbase/regionserver/HRegionServer.java | 77 ++--
.../hbase/regionserver/wal/FSWALEntry.java | 1 +
.../RecoveredReplicationSourceShipper.java | 3 +-
.../regionserver/ReplicationSourceManager.java | 121 +-
.../regionserver/ReplicationSourceShipper.java | 76 +---
.../ReplicationSourceWALReader.java | 52 +--
.../hbase/snapshot/RestoreSnapshotHelper.java | 2 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 30 --
.../java/org/apache/hadoop/hbase/wal/WAL.java | 19 +-
.../hadoop/hbase/TestMetaTableAccessor.java | 10 +-
.../regionserver/TestRegionServerMetrics.java | 2 +-
.../replication/TestSerialReplication.java | 400 ---
.../regionserver/TestGlobalThrottler.java | 2 +-
src/main/asciidoc/_chapters/ops_mgt.adoc| 41 +-
26 files changed, 87 insertions(+), 1383 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index c9807c3..e512b2c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -538,14 +538,6 @@ public class HTableDescriptor implements TableDescriptor,
Comparablehttp://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index dad9aef..7d00f92 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -17,13 +17,14 @@
*/
package org.apache.hadoop.hbase;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
import java.io.Closeable;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -33,7 +34,6 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell.Type;
import org.apache.hadoop.hbase.client.Connection;
@@ -71,9 +71,8 @@ import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import edu.umd.cs.findbugs.annotations.NonNull;
-import edu.umd.cs.findbugs.annotations.Nullable;
/**
* Read/write operations on region and assignment information store in
@@ -123,34 +122,14 @@ public class MetaTableAccessor {
* region is the result of a merge
* info:mergeB => contains a serialized HRI for the second
parent region if the
* region is the result of a merge
+ *
* The actual layout of meta should be en
[19/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b75cffe0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b75cffe0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b75cffe0
Branch: refs/heads/HBASE-19397-branch-2
Commit: b75cffe07cbfa6eb1d80c771819ab75ff83485a6
Parents: d8f09f1
Author: zhangduo
Authored: Mon Dec 25 18:49:56 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 19 +-
.../replication/ReplicationPeersZKImpl.java | 21 +-
.../replication/ReplicationQueueStorage.java| 26 +-
.../replication/ReplicationQueuesClient.java| 93 -
.../ReplicationQueuesClientArguments.java | 40 --
.../ReplicationQueuesClientZKImpl.java | 181 -
.../replication/ZKReplicationQueueStorage.java | 90 -
.../replication/TestReplicationStateBasic.java | 378 +++
.../replication/TestReplicationStateZKImpl.java | 153
.../TestZKReplicationQueueStorage.java | 74
.../cleaner/ReplicationZKNodeCleaner.java | 71 ++--
.../cleaner/ReplicationZKNodeCleanerChore.java | 5 +-
.../replication/ReplicationPeerManager.java | 31 +-
.../master/ReplicationHFileCleaner.java | 109 ++
.../master/ReplicationLogCleaner.java | 44 +--
.../regionserver/DumpReplicationQueues.java | 78 ++--
.../hbase/util/hbck/ReplicationChecker.java | 14 +-
.../client/TestAsyncReplicationAdminApi.java| 31 +-
.../replication/TestReplicationAdmin.java | 2 +
.../hbase/master/cleaner/TestLogsCleaner.java | 81 ++--
.../cleaner/TestReplicationHFileCleaner.java| 29 --
.../cleaner/TestReplicationZKNodeCleaner.java | 12 +-
.../replication/TestReplicationStateBasic.java | 378 ---
.../replication/TestReplicationStateZKImpl.java | 232
.../TestReplicationSourceManagerZkImpl.java | 41 --
25 files changed, 890 insertions(+), 1343 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 9f4ad18..6c1c213 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,20 +36,14 @@ public class ReplicationFactory {
args);
}
- public static ReplicationQueuesClient
- getReplicationQueuesClient(ReplicationQueuesClientArguments args) throws
Exception {
-return (ReplicationQueuesClient) ConstructorUtils
-.invokeConstructor(ReplicationQueuesClientZKImpl.class, args);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- Abortable abortable) {
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
}
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- final
ReplicationQueuesClient queuesClient, Abortable abortable) {
-return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ ReplicationQueueStorage queueStorage, Abortable abortable) {
+return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable);
}
public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 419e289..4e5f757 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/h
[25/50] [abbrv] hbase git commit: HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure
HBASE-19536 Client side changes for moving peer modification from zk watcher to
procedure
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29af06ab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29af06ab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29af06ab
Branch: refs/heads/HBASE-19397-branch-2
Commit: 29af06ab4b4d3ca7e6d7ba1597d176b97a9f863a
Parents: febfe26
Author: Guanghao Zhang
Authored: Tue Dec 19 15:50:57 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../org/apache/hadoop/hbase/client/Admin.java | 87 ++-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 149 ++-
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 82 +-
3 files changed, 238 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/29af06ab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 40dac2f..b8546fa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2473,7 +2473,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @throws IOException if a remote or network exception occurs
*/
default void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig)
@@ -2484,7 +2484,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @param enabled peer state, true if ENABLED and false if DISABLED
* @throws IOException if a remote or network exception occurs
*/
@@ -2492,6 +2492,37 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ default Future addReplicationPeerAsync(String peerId,
ReplicationPeerConfig peerConfig)
+ throws IOException {
+return addReplicationPeerAsync(peerId, peerConfig, true);
+ }
+
+ /**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @param enabled peer state, true if ENABLED and false if DISABLED
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future addReplicationPeerAsync(String peerId, ReplicationPeerConfig
peerConfig,
+ boolean enabled) throws IOException;
+
+ /**
* Remove a peer and stop the replication.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2499,6 +2530,18 @@ public interface Admin extends Abortable, Closeable {
void removeReplicationPeer(String peerId) throws IOException;
/**
+ * Remove a replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a shor
[38/50] [abbrv] hbase git commit: HBASE-19634 Add permission check for executeProcedures in AccessController
HBASE-19634 Add permission check for executeProcedures in AccessController
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f42fc70b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f42fc70b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f42fc70b
Branch: refs/heads/HBASE-19397-branch-2
Commit: f42fc70b13b8f4009eb4cac6d745eb36bc22f770
Parents: d448c01
Author: zhangduo
Authored: Thu Jan 4 16:18:21 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:03 2018 +0800
--
.../hbase/coprocessor/RegionServerObserver.java | 14 ++
.../hbase/regionserver/RSRpcServices.java | 52 +++-
.../RegionServerCoprocessorHost.java| 18 +++
.../hbase/security/access/AccessController.java | 30 ++-
.../hadoop/hbase/TestJMXConnectorServer.java| 7 +++
.../security/access/TestAccessController.java | 18 +--
6 files changed, 100 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f42fc70b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
index c1af3fb..5b751df 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
@@ -126,4 +126,18 @@ public interface RegionServerObserver {
default void postClearCompactionQueues(
final ObserverContext ctx)
throws IOException {}
+
+ /**
+ * This will be called before executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
preExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
+
+ /**
+ * This will be called after executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
postExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f42fc70b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 74f056b..88ce346 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -3508,36 +3508,40 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
}
@Override
+ @QosPriority(priority = HConstants.ADMIN_QOS)
public ExecuteProceduresResponse executeProcedures(RpcController controller,
ExecuteProceduresRequest request) throws ServiceException {
-if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req : request.getOpenRegionList()) {
-openRegion(controller, req);
+try {
+ checkOpen();
+ regionServer.getRegionServerCoprocessorHost().preExecuteProcedures();
+ if (request.getOpenRegionCount() > 0) {
+for (OpenRegionRequest req : request.getOpenRegionList()) {
+ openRegion(controller, req);
+}
}
-}
-if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req : request.getCloseRegionList()) {
-closeRegion(controller, req);
+ if (request.getCloseRegionCount() > 0) {
+for (CloseRegionRequest req : request.getCloseRegionList()) {
+ closeRegion(controller, req);
+}
}
-}
-if (request.getProcCount() > 0) {
- for (RemoteProcedureRequest req : request.getProcList()) {
-RSProcedureCallable callable;
-try {
- callable =
-
Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
-} catch (Exception e) {
- // here we just ignore the error as this should not happen and we do
not provide a general
- // way to report errors for all types of remote procedure. The
procedure will hang at
- // master side but after you solve the problem and restart master it
will be executed
- // again and pass.
- LOG.warn("create procedure of type " + req.getProcClass() + "
failed, give up", e);
- continue;
+ if (request.getProcCount() > 0) {
+for (RemoteProcedureRequest req : request.getProcList()) {
+
[04/50] [abbrv] hbase git commit: Revert "HBASE-19835 Use explicit casting to avoid side effects"
Revert "HBASE-19835 Use explicit casting to avoid side effects"
This reverts commit bc080e75000aa16ee05d42033ec5e4d984985205.
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
patch reverted changes that happened in parallel without explanation. see jira.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a66fa85a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a66fa85a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a66fa85a
Branch: refs/heads/HBASE-19397-branch-2
Commit: a66fa85a1650808658db9556dbdf1fac5745c9b2
Parents: d6d1e49
Author: Sean Busbey
Authored: Fri Feb 23 19:46:12 2018 -0600
Committer: Sean Busbey
Committed: Fri Feb 23 23:04:34 2018 -0600
--
.../org/apache/hadoop/hbase/client/Append.java | 2 +-
.../apache/hadoop/hbase/client/Mutation.java| 6 +-
.../hadoop/hbase/IndividualBytesFieldCell.java | 2 +-
.../apache/hadoop/hbase/PrivateCellUtil.java| 4 +-
.../hbase/regionserver/CompactionPipeline.java | 2 +-
.../hadoop/hbase/regionserver/HRegion.java | 4 +-
.../hadoop/hbase/regionserver/HStore.java | 94 +++-
.../hbase/regionserver/MemStoreCompactor.java | 2 +-
.../MultiVersionConcurrencyControl.java | 2 +-
.../hadoop/hbase/regionserver/ScanInfo.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 2 +-
.../apache/hadoop/hbase/wal/WALSplitter.java| 2 +-
12 files changed, 67 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
index 042209c..61474b7 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
@@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Public
public class Append extends Mutation {
private static final Logger LOG = LoggerFactory.getLogger(Append.class);
- private static final long HEAP_OVERHEAD = (long)ClassSize.REFERENCE +
ClassSize.TIMERANGE;
+ private static final long HEAP_OVERHEAD = ClassSize.REFERENCE +
ClassSize.TIMERANGE;
private TimeRange tr = new TimeRange();
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index 09000ac..1338fff 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
@@ -66,7 +66,7 @@ public abstract class Mutation extends
OperationWithAttributes implements Row, C
HeapSize {
public static final long MUTATION_OVERHEAD = ClassSize.align(
// This
- (long)ClassSize.OBJECT +
+ ClassSize.OBJECT +
// row + OperationWithAttributes.attributes
2 * ClassSize.REFERENCE +
// Timestamp
@@ -791,10 +791,10 @@ public abstract class Mutation extends
OperationWithAttributes implements Row, C
private static final class CellWrapper implements ExtendedCell {
private static final long FIXED_OVERHEAD = ClassSize.align(
- (long)ClassSize.OBJECT // object header
+ ClassSize.OBJECT // object header
+ KeyValue.TIMESTAMP_SIZE // timestamp
+ Bytes.SIZEOF_LONG // sequence id
-+ 1L * ClassSize.REFERENCE); // references to cell
++ 1 * ClassSize.REFERENCE); // references to cell
private final Cell cell;
private long sequenceId;
private long timestamp;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
index 6a95a7c..7093b4b 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
@@ -27,7 +27,7 @@ import org.apache.yetus.audience.Inte
[29/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
HBASE-19622 Reimplement ReplicationPeers with the new replication storage
interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fef99645
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fef99645
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fef99645
Branch: refs/heads/HBASE-19397-branch-2
Commit: fef996454fc5e44878cf972f744920358d20ebf1
Parents: df6fc97
Author: huzheng
Authored: Tue Dec 26 16:46:10 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:24 2018 +0800
--
.../replication/ReplicationPeerConfigUtil.java | 10 +-
.../replication/VerifyReplication.java | 9 +-
.../hbase/replication/ReplicationFactory.java | 10 +-
.../hbase/replication/ReplicationPeerImpl.java | 60 +-
.../replication/ReplicationPeerStorage.java | 3 +-
.../hbase/replication/ReplicationPeers.java | 238
.../replication/ReplicationPeersZKImpl.java | 552 ---
.../replication/ZKReplicationPeerStorage.java | 12 +-
.../replication/ZKReplicationStorageBase.java | 3 +-
.../replication/TestReplicationStateBasic.java | 125 ++---
.../replication/TestReplicationStateZKImpl.java | 2 +-
.../TestZKReplicationPeerStorage.java | 12 +-
.../cleaner/ReplicationZKNodeCleaner.java | 57 +-
.../replication/ReplicationPeerManager.java | 6 +-
.../regionserver/DumpReplicationQueues.java | 2 +-
.../regionserver/PeerProcedureHandlerImpl.java | 49 +-
.../replication/regionserver/Replication.java | 2 +-
.../regionserver/ReplicationSource.java | 7 +-
.../regionserver/ReplicationSourceManager.java | 44 +-
.../cleaner/TestReplicationHFileCleaner.java| 7 +-
.../replication/TestMultiSlaveReplication.java | 2 -
.../TestReplicationTrackerZKImpl.java | 26 +-
.../TestReplicationSourceManager.java | 17 +-
.../hadoop/hbase/HBaseZKTestingUtility.java | 3 +-
24 files changed, 307 insertions(+), 951 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 022bf64..a234a9b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -247,22 +247,22 @@ public final class ReplicationPeerConfigUtil {
public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
throws DeserializationException {
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
- int pblen = ProtobufUtil.lengthOfPBMagic();
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
ReplicationProtos.ReplicationPeer.Builder builder =
ReplicationProtos.ReplicationPeer.newBuilder();
ReplicationProtos.ReplicationPeer peer;
try {
-ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
peer = builder.build();
} catch (IOException e) {
throw new DeserializationException(e);
}
return convert(peer);
} else {
- if (bytes.length > 0) {
-return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
+ if (bytes == null || bytes.length <= 0) {
+throw new DeserializationException("Bytes to deserialize should not be
empty.");
}
- return ReplicationPeerConfig.newBuilder().setClusterKey("").build();
+ return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 09d4b4b..f0070f0 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -339,15 +339,10 @@ public class VerifyReplication extends Configured
implements Tool {
@Override public boolea
[24/50] [abbrv] hbase git commit: HBASE-19564 Procedure id is missing in the response of peer related operations
HBASE-19564 Procedure id is missing in the response of peer related operations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50513f84
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50513f84
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50513f84
Branch: refs/heads/HBASE-19397-branch-2
Commit: 50513f845e1470d89bd84621b371579d2f0616dc
Parents: 29af06a
Author: zhangduo
Authored: Wed Dec 20 20:57:37 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hadoop/hbase/master/MasterRpcServices.java | 24 ++--
.../master/replication/ModifyPeerProcedure.java | 4 +---
2 files changed, 13 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/50513f84/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 3c24e4d..4622a00 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1866,10 +1866,10 @@ public class MasterRpcServices extends RSRpcServices
public AddReplicationPeerResponse addReplicationPeer(RpcController
controller,
AddReplicationPeerRequest request) throws ServiceException {
try {
- master.addReplicationPeer(request.getPeerId(),
-ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
request.getPeerState()
-.getState().equals(ReplicationState.State.ENABLED));
- return AddReplicationPeerResponse.newBuilder().build();
+ long procId = master.addReplicationPeer(request.getPeerId(),
+ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
+
request.getPeerState().getState().equals(ReplicationState.State.ENABLED));
+ return AddReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1879,8 +1879,8 @@ public class MasterRpcServices extends RSRpcServices
public RemoveReplicationPeerResponse removeReplicationPeer(RpcController
controller,
RemoveReplicationPeerRequest request) throws ServiceException {
try {
- master.removeReplicationPeer(request.getPeerId());
- return RemoveReplicationPeerResponse.newBuilder().build();
+ long procId = master.removeReplicationPeer(request.getPeerId());
+ return
RemoveReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1890,8 +1890,8 @@ public class MasterRpcServices extends RSRpcServices
public EnableReplicationPeerResponse enableReplicationPeer(RpcController
controller,
EnableReplicationPeerRequest request) throws ServiceException {
try {
- master.enableReplicationPeer(request.getPeerId());
- return EnableReplicationPeerResponse.newBuilder().build();
+ long procId = master.enableReplicationPeer(request.getPeerId());
+ return
EnableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1901,8 +1901,8 @@ public class MasterRpcServices extends RSRpcServices
public DisableReplicationPeerResponse disableReplicationPeer(RpcController
controller,
DisableReplicationPeerRequest request) throws ServiceException {
try {
- master.disableReplicationPeer(request.getPeerId());
- return DisableReplicationPeerResponse.newBuilder().build();
+ long procId = master.disableReplicationPeer(request.getPeerId());
+ return
DisableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1928,9 +1928,9 @@ public class MasterRpcServices extends RSRpcServices
public UpdateReplicationPeerConfigResponse
updateReplicationPeerConfig(RpcController controller,
UpdateReplicationPeerConfigRequest request) throws ServiceException {
try {
- master.updateReplicationPeerConfig(request.getPeerId(),
+ long procId = master.updateReplicationPeerConfig(request.getPeerId(),
ReplicationPeerConfigUtil.convert(request.getPeerConfig()));
- return UpdateReplicationPeerConfigResponse.newBuilder().build();
+ return
UpdateReplicationPeerConfigResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
thr
[23/50] [abbrv] hbase git commit: HBASE-19592 Add UTs to test retry on update zk failure
HBASE-19592 Add UTs to test retry on update zk failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b13faf0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b13faf0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b13faf0
Branch: refs/heads/HBASE-19397-branch-2
Commit: 3b13faf09557074c9ff52714f20cea31ffd483b3
Parents: b3be773
Author: zhangduo
Authored: Tue Dec 26 20:39:00 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../replication/ReplicationPeerManager.java | 5 +-
.../TestReplicationProcedureRetry.java | 200 +++
2 files changed, 202 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b13faf0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index b78cbce..f4ccce8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Used to add/remove a replication peer.
*/
@InterfaceAudience.Private
-public final class ReplicationPeerManager {
+public class ReplicationPeerManager {
private final ReplicationPeerStorage peerStorage;
@@ -61,8 +61,7 @@ public final class ReplicationPeerManager {
private final ConcurrentMap peers;
- private ReplicationPeerManager(ReplicationPeerStorage peerStorage,
- ReplicationQueueStorage queueStorage,
+ ReplicationPeerManager(ReplicationPeerStorage peerStorage,
ReplicationQueueStorage queueStorage,
ConcurrentMap peers) {
this.peerStorage = peerStorage;
this.queueStorage = queueStorage;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b13faf0/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
new file mode 100644
index 000..ab35b46
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+
+/**
+ * All the modification method will fail once in the test and should finally
succeed.
+ */
+@Category({ Replic
[27/50] [abbrv] hbase git commit: HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df6fc97f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df6fc97f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df6fc97f
Branch: refs/heads/HBASE-19397-branch-2
Commit: df6fc97f5f60db420af7f7ce63461e40b8bec3f8
Parents: 6bbc607
Author: zhangduo
Authored: Wed Dec 27 20:13:42 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:24 2018 +0800
--
.../src/main/protobuf/RegionServerStatus.proto | 5 +-
.../hadoop/hbase/master/MasterRpcServices.java | 15 ++-
.../hbase/regionserver/HRegionServer.java | 72
.../RemoteProcedureResultReporter.java | 111 +++
.../handler/RSProcedureHandler.java | 2 +-
5 files changed, 149 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 4f75941..3f836cd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -146,7 +146,7 @@ message RegionSpaceUseReportRequest {
message RegionSpaceUseReportResponse {
}
-message ReportProcedureDoneRequest {
+message RemoteProcedureResult {
required uint64 proc_id = 1;
enum Status {
SUCCESS = 1;
@@ -155,6 +155,9 @@ message ReportProcedureDoneRequest {
required Status status = 2;
optional ForeignExceptionMessage error = 3;
}
+message ReportProcedureDoneRequest {
+ repeated RemoteProcedureResult result = 1;
+}
message ReportProcedureDoneResponse {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 4622a00..b4f0faf 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -266,6 +266,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
@@ -2234,12 +2235,14 @@ public class MasterRpcServices extends RSRpcServices
@Override
public ReportProcedureDoneResponse reportProcedureDone(RpcController
controller,
ReportProcedureDoneRequest request) throws ServiceException {
-if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
- master.remoteProcedureCompleted(request.getProcId());
-} else {
- master.remoteProcedureFailed(request.getProcId(),
-RemoteProcedureException.fromProto(request.getError()));
-}
+request.getResultList().forEach(result -> {
+ if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) {
+master.remoteProcedureCompleted(result.getProcId());
+ } else {
+master.remoteProcedureFailed(result.getProcId(),
+ RemoteProcedureException.fromProto(result.getError()));
+ }
+});
return ReportProcedureDoneResponse.getDefaultInstance();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c95ac37..81febc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/region
[39/50] [abbrv] hbase git commit: HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/85e85bea
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/85e85bea
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/85e85bea
Branch: refs/heads/HBASE-19397-branch-2
Commit: 85e85bea03e57b766d251874740cb4cdbc981abd
Parents: 718e664
Author: huzheng
Authored: Fri Dec 29 15:55:28 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:03 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 5 +-
.../replication/ReplicationStateZKBase.java | 159 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +--
.../replication/ZKReplicationPeerStorage.java | 24 ++-
.../replication/ZKReplicationStorageBase.java | 13 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
.../master/ReplicationPeerConfigUpgrader.java | 128 +++
.../regionserver/DumpReplicationQueues.java | 18 +--
.../replication/regionserver/Replication.java | 3 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +-
.../TestReplicationTrackerZKImpl.java | 3 +-
.../replication/master/TestTableCFsUpdater.java | 41 ++---
.../TestReplicationSourceManager.java | 6 +-
13 files changed, 136 insertions(+), 292 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/85e85bea/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c66aff..2a970ba 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -33,9 +33,8 @@ public class ReplicationFactory {
return new ReplicationPeers(zk, conf);
}
- public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable
abortable,
+ public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
Abortable abortable,
Stoppable stopper) {
-return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf,
abortable, stopper);
+return new ReplicationTrackerZKImpl(zookeeper, abortable, stopper);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/85e85bea/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
deleted file mode 100644
index a48683e..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audi
[44/50] [abbrv] hbase git commit: HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint
UT hangs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b980c46
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b980c46
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b980c46
Branch: refs/heads/HBASE-19397-branch-2
Commit: 0b980c4603366e62dfbb1e56a17ac39693ddbbc3
Parents: f8b58c3
Author: huzheng
Authored: Wed Jan 10 15:00:30 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b980c46/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 45058a1..f55d065 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -40,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -71,6 +70,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
@@ -3409,13 +3409,12 @@ public class HMaster extends HRegionServer implements
MasterServices {
cpHost.preGetReplicationPeerConfig(peerId);
}
LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" +
peerId);
-Optional peerConfig =
- this.replicationPeerManager.getPeerConfig(peerId);
-
+ReplicationPeerConfig peerConfig =
this.replicationPeerManager.getPeerConfig(peerId)
+.orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
if (cpHost != null) {
cpHost.postGetReplicationPeerConfig(peerId);
}
-return peerConfig.orElse(null);
+return peerConfig;
}
@Override
[45/50] [abbrv] hbase git commit: HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/03984c67
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/03984c67
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/03984c67
Branch: refs/heads/HBASE-19397-branch-2
Commit: 03984c6787e98fe2bf96a7817b9318fdbd6c962f
Parents: 8cac522
Author: Guanghao Zhang
Authored: Fri Jan 5 15:39:06 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 23
1 file changed, 19 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/03984c67/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index b18dd6c..eab06a2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -290,7 +290,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
@Override
public void completionCleanup(final Procedure proc) {
if (proc instanceof TableProcedureInterface) {
- TableProcedureInterface iProcTable = (TableProcedureInterface)proc;
+ TableProcedureInterface iProcTable = (TableProcedureInterface) proc;
boolean tableDeleted;
if (proc.hasException()) {
Exception procEx = proc.getException().unwrapRemoteException();
@@ -311,9 +311,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
}
} else if (proc instanceof PeerProcedureInterface) {
PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc;
- if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) {
-removePeerQueue(iProcPeer.getPeerId());
- }
+ tryCleanupPeerQueue(iProcPeer.getPeerId(), proc);
} else {
// No cleanup for ServerProcedureInterface types, yet.
return;
@@ -402,6 +400,23 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
locking.removePeerLock(peerId);
}
+ private void tryCleanupPeerQueue(String peerId, Procedure procedure) {
+schedLock();
+try {
+ PeerQueue queue = AvlTree.get(peerMap, peerId,
PEER_QUEUE_KEY_COMPARATOR);
+ if (queue == null) {
+return;
+ }
+
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
+removeFromRunQueue(peerRunQueue, queue);
+removePeerQueue(peerId);
+ }
+} finally {
+ schedUnlock();
+}
+ }
private static boolean isPeerProcedure(Procedure proc) {
return proc instanceof PeerProcedureInterface;
[36/50] [abbrv] hbase git commit: HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore
HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker
and remove ReplicationZKNodeCleanerChore
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/718e664b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/718e664b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/718e664b
Branch: refs/heads/HBASE-19397-branch-2
Commit: 718e664b707745d18226eae90796d7c0965c7534
Parents: 1d7960c
Author: zhangduo
Authored: Wed Jan 3 09:39:44 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:03 2018 +0800
--
.../replication/VerifyReplication.java | 6 +-
.../hbase/replication/ReplicationPeers.java | 26 +--
.../hbase/replication/ReplicationUtils.java | 38
.../replication/TestReplicationStateBasic.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 13 --
.../cleaner/ReplicationZKNodeCleaner.java | 192 ---
.../cleaner/ReplicationZKNodeCleanerChore.java | 54 --
.../replication/ReplicationPeerManager.java | 18 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 11 +-
.../hbase/util/hbck/ReplicationChecker.java | 109 +++
.../cleaner/TestReplicationZKNodeCleaner.java | 115 ---
.../hbase/util/TestHBaseFsckReplication.java| 101 ++
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 6 +-
13 files changed, 226 insertions(+), 465 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/718e664b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index fe45762..fac4875 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -345,10 +345,10 @@ public class VerifyReplication extends Configured
implements Tool {
}
});
ReplicationPeerStorage storage =
- ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
+ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId);
return Pair.newPair(peerConfig,
-ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf));
+ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf));
} catch (ReplicationException e) {
throw new IOException("An error occurred while trying to connect to the
remove peer cluster",
e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/718e664b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 45940a5..fcbc350 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -17,14 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -106,25 +103,6 @@ public class ReplicationPeers {
return Collections.unmodifiableSet(peerCache.keySet());
}
- public
[34/50] [abbrv] hbase git commit: HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a75ee53d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a75ee53d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a75ee53d
Branch: refs/heads/HBASE-19397-branch-2
Commit: a75ee53d8440d9ca71a20587fb34c558c97ba6f9
Parents: 8a762b6
Author: zhangduo
Authored: Tue Jan 2 16:13:55 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:58 2018 +0800
--
.../regionserver/PeerProcedureHandlerImpl.java | 41 ++--
1 file changed, 29 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a75ee53d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 1efe180..c09c6a0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import java.util.concurrent.locks.ReentrantLock;
-
+import java.util.concurrent.locks.Lock;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.KeyLocker;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +32,7 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private static final Logger LOG =
LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private final ReplicationSourceManager replicationSourceManager;
- private final ReentrantLock peersLock = new ReentrantLock();
+ private final KeyLocker peersLock = new KeyLocker<>();
public PeerProcedureHandlerImpl(ReplicationSourceManager
replicationSourceManager) {
this.replicationSourceManager = replicationSourceManager;
@@ -40,40 +40,57 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void addPeer(String peerId) throws ReplicationException, IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
replicationSourceManager.addPeer(peerId);
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void removePeer(String peerId) throws ReplicationException,
IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
if (replicationSourceManager.getReplicationPeers().getPeer(peerId) !=
null) {
replicationSourceManager.removePeer(peerId);
}
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void disablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("disable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("disable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("enable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("enable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
-replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+} finally {
+ peerLock.unlock(
[43/50] [abbrv] hbase git commit: HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed
HBASE-19783 Change replication peer cluster key/endpoint from a not-null value
to null is not allowed
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c26916dd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c26916dd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c26916dd
Branch: refs/heads/HBASE-19397-branch-2
Commit: c26916ddb78c66563974113d649e4af430627394
Parents: 0b980c4
Author: Guanghao Zhang
Authored: Fri Jan 12 22:04:38 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../replication/ReplicationPeerManager.java | 28 +---
1 file changed, 19 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c26916dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 696b2d7..19fc7f4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -132,20 +132,19 @@ public class ReplicationPeerManager {
checkPeerConfig(peerConfig);
ReplicationPeerDescription desc = checkPeerExists(peerId);
ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
-if (!StringUtils.isBlank(peerConfig.getClusterKey()) &&
- !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) {
+if (!isStringEquals(peerConfig.getClusterKey(),
oldPeerConfig.getClusterKey())) {
throw new DoNotRetryIOException(
"Changing the cluster key on an existing peer is not allowed.
Existing key '" +
-oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
-peerConfig.getClusterKey() + "'");
+ oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
+ peerConfig.getClusterKey() + "'");
}
-if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) &&
-
!peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl()))
{
+if (!isStringEquals(peerConfig.getReplicationEndpointImpl(),
+ oldPeerConfig.getReplicationEndpointImpl())) {
throw new DoNotRetryIOException("Changing the replication endpoint
implementation class " +
-"on an existing peer is not allowed. Existing class '" +
-oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
-" does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
+ "on an existing peer is not allowed. Existing class '" +
+ oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
+ " does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
}
}
@@ -341,4 +340,15 @@ public class ReplicationPeerManager {
return new ReplicationPeerManager(peerStorage,
ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
}
+
+ /**
+ * For replication peer cluster key or endpoint class, null and empty string
is same. So here
+ * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly.
+ */
+ private boolean isStringEquals(String s1, String s2) {
+if (StringUtils.isBlank(s1)) {
+ return StringUtils.isBlank(s2);
+}
+return s1.equals(s2);
+ }
}
[03/50] [abbrv] hbase git commit: HBASE-20054 removes the indefinite retry introduced by HBASE-18282
HBASE-20054 removes the indefinite retry introduced by HBASE-18282
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d6d1e49a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d6d1e49a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d6d1e49a
Branch: refs/heads/HBASE-19397-branch-2
Commit: d6d1e49a7561ac1b13d965bf74436919de625f1d
Parents: 4ddfdaf
Author: tedyu
Authored: Fri Feb 23 19:14:04 2018 -0800
Committer: tedyu
Committed: Fri Feb 23 19:14:04 2018 -0800
--
.../regionserver/ReplicationSourceManager.java | 19 +--
1 file changed, 5 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d6d1e49a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 2cf3a82..bfd338d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -782,20 +782,11 @@ public class ReplicationSourceManager implements
ReplicationListener {
@Override
public void run() {
List currentReplicators = null;
- while (currentReplicators == null) {
-try {
- currentReplicators = replicationQueues.getListOfReplicators();
-} catch (ReplicationException e1) {
- LOG.warn("Failure in getListOfReplicators(), will retry later", e1);
- try {
-Thread.sleep(ThreadLocalRandom.current().nextInt(1));
- } catch (InterruptedException e2) {
-LOG.warn("Interrupted while waiting for list of replicators to be
available, "
-+ "will not adopt any abandoned queues", e2);
-Thread.currentThread().interrupt();
-break;
- }
-}
+ try {
+currentReplicators = replicationQueues.getListOfReplicators();
+ } catch (ReplicationException e) {
+server.abort("Failed to get all replicators", e);
+return;
}
if (currentReplicators == null || currentReplicators.isEmpty()) {
return;
[22/50] [abbrv] hbase git commit: HBASE-19642 Fix locking for peer modification procedure
HBASE-19642 Fix locking for peer modification procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/296d5868
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/296d5868
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/296d5868
Branch: refs/heads/HBASE-19397-branch-2
Commit: 296d58682c083417b87e91acdfcbb30cdb8b0593
Parents: 3b13faf
Author: zhangduo
Authored: Wed Dec 27 18:27:13 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../master/replication/ModifyPeerProcedure.java | 21 +---
1 file changed, 18 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/296d5868/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 279fbc7..a682606 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -46,6 +46,8 @@ public abstract class ModifyPeerProcedure
protected String peerId;
+ private volatile boolean locked;
+
// used to keep compatible with old client where we can only returns after
updateStorage.
protected ProcedurePrepareLatch latch;
@@ -145,17 +147,30 @@ public abstract class ModifyPeerProcedure
@Override
protected LockState acquireLock(MasterProcedureEnv env) {
-return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)
- ? LockState.LOCK_EVENT_WAIT
- : LockState.LOCK_ACQUIRED;
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
}
@Override
protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
}
@Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protected boolean hasLock(MasterProcedureEnv env) {
+return locked;
+ }
+
+ @Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState
state)
throws IOException, InterruptedException {
if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
[42/50] [abbrv] hbase git commit: HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint
HBASE-19707 Race in start and terminate of a replication source after we async
start replicatione endpoint
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8cac522b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8cac522b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8cac522b
Branch: refs/heads/HBASE-19397-branch-2
Commit: 8cac522bd78cfd3d7a866c552017ab71420cfb13
Parents: 24eff34
Author: zhangduo
Authored: Fri Jan 5 18:28:44 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../RecoveredReplicationSource.java | 16 +-
.../regionserver/ReplicationSource.java | 203 ++-
2 files changed, 116 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8cac522b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 1be9a88..3cae0f2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -68,7 +68,7 @@ public class RecoveredReplicationSource extends
ReplicationSource {
LOG.debug("Someone has beat us to start a worker thread for wal group "
+ walGroupId);
} else {
LOG.debug("Starting up worker for wal group " + walGroupId);
- worker.startup(getUncaughtExceptionHandler());
+ worker.startup(this::uncaughtException);
worker.setWALReader(
startNewWALReader(worker.getName(), walGroupId, queue,
worker.getStartPosition()));
workerThreads.put(walGroupId, worker);
@@ -76,13 +76,13 @@ public class RecoveredReplicationSource extends
ReplicationSource {
}
@Override
- protected ReplicationSourceWALReader startNewWALReader(String threadName,
- String walGroupId, PriorityBlockingQueue queue, long
startPosition) {
-ReplicationSourceWALReader walReader = new
RecoveredReplicationSourceWALReader(fs,
-conf, queue, startPosition, walEntryFilter, this);
-Threads.setDaemonThreadRunning(walReader, threadName
-+ ".replicationSource.replicationWALReaderThread." + walGroupId + ","
+ queueId,
- getUncaughtExceptionHandler());
+ protected ReplicationSourceWALReader startNewWALReader(String threadName,
String walGroupId,
+ PriorityBlockingQueue queue, long startPosition) {
+ReplicationSourceWALReader walReader =
+ new RecoveredReplicationSourceWALReader(fs, conf, queue, startPosition,
walEntryFilter, this);
+Threads.setDaemonThreadRunning(walReader,
+ threadName + ".replicationSource.replicationWALReaderThread." +
walGroupId + "," + queueId,
+ this::uncaughtException);
return walReader;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8cac522b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0092251..09b6cc1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -75,7 +75,7 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
*
*/
@InterfaceAudience.Private
-public class ReplicationSource extends Thread implements
ReplicationSourceInterface {
+public class ReplicationSource implements ReplicationSourceInterface {
private static final Logger LOG =
LoggerFactory.getLogger(ReplicationSource.class);
// Queues of logs to process, entry in format of walGroupId->queue,
@@ -114,10 +114,8 @@ public class ReplicationSource extends Thread implements
ReplicationSourceInterf
private MetricsSource metrics;
// WARN threshold for the number of queued logs, defaults to 2
private int logQueueWarnThreshold;
- // whether the replication endpoint has been initialized
- private volatile boolean endpointInitialized = false;
// ReplicationEndpoint which will handle the actual replication
- private ReplicationEndpoint replicationEndpoint;
+ private volatile ReplicationEndpoint replicationEndpoint;
// A filter (or a
[14/50] [abbrv] hbase git commit: HBASE-19520 Add UTs for the new lock type PEER
HBASE-19520 Add UTs for the new lock type PEER
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de68d00b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de68d00b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de68d00b
Branch: refs/heads/HBASE-19397-branch-2
Commit: de68d00bdab986af405b9d646e270445d3674fd9
Parents: 50513f8
Author: Guanghao Zhang
Authored: Wed Dec 20 16:43:38 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 9 +-
.../procedure/TestMasterProcedureScheduler.java | 65 -
...TestMasterProcedureSchedulerConcurrency.java | 135 +++
3 files changed, 201 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/de68d00b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 58f9ccd..b18dd6c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -277,6 +277,13 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
while (tableIter.hasNext()) {
count += tableIter.next().size();
}
+
+// Peer queues
+final AvlTreeIterator peerIter = new AvlTreeIterator<>(peerMap);
+while (peerIter.hasNext()) {
+ count += peerIter.next().size();
+}
+
return count;
}
@@ -807,7 +814,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
* @see #wakePeerExclusiveLock(Procedure, String)
* @param procedure the procedure trying to acquire the lock
* @param peerId peer to lock
- * @return true if the procedure has to wait for the per to be available
+ * @return true if the procedure has to wait for the peer to be available
*/
public boolean waitPeerExclusiveLock(Procedure procedure, String peerId) {
schedLock();
http://git-wip-us.apache.org/repos/asf/hbase/blob/de68d00b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index 160f4d2..65757db 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -908,6 +908,27 @@ public class TestMasterProcedureScheduler {
}
}
+ public static class TestPeerProcedure extends TestProcedure implements
PeerProcedureInterface {
+private final String peerId;
+private final PeerOperationType opType;
+
+public TestPeerProcedure(long procId, String peerId, PeerOperationType
opType) {
+ super(procId);
+ this.peerId = peerId;
+ this.opType = opType;
+}
+
+@Override
+public String getPeerId() {
+ return peerId;
+}
+
+@Override
+public PeerOperationType getPeerOperationType() {
+ return opType;
+}
+ }
+
private static LockProcedure createLockProcedure(LockType lockType, long
procId) throws Exception {
LockProcedure procedure = new LockProcedure();
@@ -930,22 +951,19 @@ public class TestMasterProcedureScheduler {
return createLockProcedure(LockType.SHARED, procId);
}
- private static void assertLockResource(LockedResource resource,
- LockedResourceType resourceType, String resourceName)
- {
+ private static void assertLockResource(LockedResource resource,
LockedResourceType resourceType,
+ String resourceName) {
assertEquals(resourceType, resource.getResourceType());
assertEquals(resourceName, resource.getResourceName());
}
- private static void assertExclusiveLock(LockedResource resource,
Procedure procedure)
- {
+ private static void assertExclusiveLock(LockedResource resource,
Procedure procedure) {
assertEquals(LockType.EXCLUSIVE, resource.getLockType());
assertEquals(procedure, resource.getExclusiveLockOwnerProcedure());
assertEquals(0, resource.getSharedLockCount());
}
- private static void assertSharedLock(LockedResource resource, int lockCount)
-
[49/50] [abbrv] hbase git commit: HBASE-19873 addendum add missing rule for new tests
HBASE-19873 addendum add missing rule for new tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/05024640
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/05024640
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/05024640
Branch: refs/heads/HBASE-19397-branch-2
Commit: 050246402987ee5611e5af129589baf431d63bd5
Parents: c26916d
Author: zhangduo
Authored: Tue Jan 30 09:40:23 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../hbase/replication/TestZKReplicationPeerStorage.java | 7 ++-
.../hbase/replication/TestZKReplicationQueueStorage.java | 8 +++-
.../hbase/replication/TestReplicationProcedureRetry.java | 7 ++-
.../apache/hadoop/hbase/util/TestHBaseFsckReplication.java | 6 ++
4 files changed, 25 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
index 3eb11da..3290fb0 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -33,19 +33,24 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Stream;
-
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationPeerStorage {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationPeerStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationPeerStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
index 786730f..2c01a26 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -27,8 +27,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
-
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -38,11 +38,17 @@ import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationQueueStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
index ab35b46..a2ae0b4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
[30/50] [abbrv] hbase git commit: HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer
HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7091095b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7091095b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7091095b Branch: refs/heads/HBASE-19397-branch-2 Commit: 7091095bd45f2a0c0bb79acd69b5aaceb1e6 Parents: fef9964 Author: zhangduo Authored: Tue Jan 2 09:57:23 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:24 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 2 +- .../replication/VerifyReplication.java | 34 ++ .../hbase/replication/ReplicationPeers.java | 32 +++-- .../replication/ZKReplicationQueueStorage.java | 3 +- .../replication/ZKReplicationStorageBase.java | 4 +-- .../replication/TestReplicationStateBasic.java | 10 ++ .../master/replication/AddPeerProcedure.java| 5 ++- .../replication/DisablePeerProcedure.java | 3 +- .../master/replication/EnablePeerProcedure.java | 3 +- .../master/replication/ModifyPeerProcedure.java | 34 ++ .../replication/RefreshPeerProcedure.java | 17 - .../master/replication/RemovePeerProcedure.java | 7 ++-- .../replication/ReplicationPeerManager.java | 31 +++- .../replication/UpdatePeerConfigProcedure.java | 3 +- .../RemoteProcedureResultReporter.java | 3 +- .../regionserver/RefreshPeerCallable.java | 5 +-- .../regionserver/ReplicationSourceManager.java | 38 +++- .../TestReplicationAdminUsingProcedure.java | 7 ++-- 18 files changed, 124 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7091095b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index b80ee16..fdae288 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -27,8 +27,8 @@ import java.util.Set; import java.util.TreeMap; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; /** * A configuration for the replication peer cluster. http://git-wip-us.apache.org/repos/asf/hbase/blob/7091095b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index f0070f0..fe45762 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.mapreduce.replication; import java.io.IOException; import java.util.Arrays; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -45,13 +44,14 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -66,6 +66,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; imp
[26/50] [abbrv] hbase git commit: HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure
HBASE-19524 Master side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/febfe262
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/febfe262
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/febfe262
Branch: refs/heads/HBASE-19397-branch-2
Commit: febfe262446a999e785a6b2bed1a7cc7437848b0
Parents: 74f9e73
Author: zhangduo
Authored: Mon Dec 18 15:22:36 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../procedure2/RemoteProcedureDispatcher.java | 3 +-
.../src/main/protobuf/MasterProcedure.proto | 21 -
.../src/main/protobuf/RegionServerStatus.proto | 3 +-
.../src/main/protobuf/Replication.proto | 5 +
.../replication/ReplicationPeersZKImpl.java | 4 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 95 ---
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/master/MasterServices.java | 26 +++---
.../assignment/RegionTransitionProcedure.java | 13 +--
.../master/procedure/MasterProcedureEnv.java| 5 +
.../master/procedure/ProcedurePrepareLatch.java | 2 +-
.../master/replication/AddPeerProcedure.java| 97
.../replication/DisablePeerProcedure.java | 70 ++
.../master/replication/EnablePeerProcedure.java | 69 ++
.../master/replication/ModifyPeerProcedure.java | 97 +---
.../master/replication/RefreshPeerCallable.java | 67 --
.../replication/RefreshPeerProcedure.java | 28 --
.../master/replication/RemovePeerProcedure.java | 69 ++
.../master/replication/ReplicationManager.java | 76 ---
.../replication/UpdatePeerConfigProcedure.java | 92 +++
.../hbase/regionserver/HRegionServer.java | 5 +-
.../regionserver/RefreshPeerCallable.java | 70 ++
.../hbase/master/MockNoopMasterServices.java| 23 +++--
.../replication/DummyModifyPeerProcedure.java | 13 ++-
24 files changed, 733 insertions(+), 224 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 7e3dde6..fb852c3 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -247,9 +247,8 @@ public abstract class RemoteProcedureDispatcherhttp://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index d587ebf..e45e12f 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -27,6 +27,7 @@ option optimize_for = SPEED;
import "HBase.proto";
import "RPC.proto";
import "Snapshot.proto";
+import "Replication.proto";
//
// WARNING - Compatibility rules
@@ -368,9 +369,10 @@ message GCMergedRegionsStateData {
}
enum PeerModificationState {
- UPDATE_PEER_STORAGE = 1;
- REFRESH_PEER_ON_RS = 2;
- POST_PEER_MODIFICATION = 3;
+ PRE_PEER_MODIFICATION = 1;
+ UPDATE_PEER_STORAGE = 2;
+ REFRESH_PEER_ON_RS = 3;
+ POST_PEER_MODIFICATION = 4;
}
message PeerModificationStateData {
@@ -395,4 +397,17 @@ message RefreshPeerParameter {
required string peer_id = 1;
required PeerModificationType type = 2;
required ServerName target_server = 3;
+}
+
+message ModifyPeerStateData {
+ required string peer_id = 1;
+}
+
+message AddPeerStateData {
+ required ReplicationPeer peer_config = 1;
+ required bool enabled = 2;
+}
+
+message UpdatePeerConfigStateData {
+ required ReplicationPeer peer_config = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index eb396ac..4f75941 100644
--- a/hbase-protocol-sh
[05/50] [abbrv] hbase git commit: HBASE-20061 HStore synchronized member variable filesCompacting should be private
HBASE-20061 HStore synchronized member variable filesCompacting should be private Signed-off-by: Chia-Ping Tsai Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/62b8da92 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/62b8da92 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/62b8da92 Branch: refs/heads/HBASE-19397-branch-2 Commit: 62b8da927e71af5489165b114481d0a62e58fba6 Parents: a66fa85 Author: Sean Busbey Authored: Fri Feb 23 14:38:03 2018 -0600 Committer: Sean Busbey Committed: Sat Feb 24 10:58:14 2018 -0600 -- .../main/java/org/apache/hadoop/hbase/regionserver/HStore.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/62b8da92/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index f20972e..220881d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -180,8 +180,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat private ScanInfo scanInfo; + // All access must be synchronized. // TODO: ideally, this should be part of storeFileManager, as we keep passing this to it. - final List filesCompacting = Lists.newArrayList(); + private final List filesCompacting = Lists.newArrayList(); // All access must be synchronized. private final Set changedReaderObservers =
[47/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
HBASE-19636 All rs should already start work with the new peer change when
replication peer procedure is finished
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/24eff344
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/24eff344
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/24eff344
Branch: refs/heads/HBASE-19397-branch-2
Commit: 24eff344ddc3541b7fa32401012a55861c432081
Parents: f42fc70
Author: Guanghao Zhang
Authored: Thu Jan 4 16:58:01 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:58:44 2018 +0800
--
.../replication/ReplicationPeerConfig.java | 1 -
.../hbase/replication/ReplicationPeerImpl.java | 4 +-
.../hbase/replication/ReplicationQueueInfo.java | 23 +-
.../hbase/replication/ReplicationUtils.java | 56 ++
.../replication/TestReplicationStateZKImpl.java | 21 -
.../regionserver/ReplicationSourceService.java | 3 +-
.../regionserver/PeerProcedureHandler.java | 3 +
.../regionserver/PeerProcedureHandlerImpl.java | 50 +-
.../RecoveredReplicationSource.java | 6 +-
.../RecoveredReplicationSourceShipper.java | 8 +-
.../replication/regionserver/Replication.java | 11 +-
.../regionserver/ReplicationSource.java | 34 +-
.../regionserver/ReplicationSourceFactory.java | 4 +-
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 827 ++-
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALReader.java | 2 +-
.../replication/ReplicationSourceDummy.java | 2 +-
.../replication/TestNamespaceReplication.java | 57 +-
.../TestReplicationSourceManager.java | 5 +-
20 files changed, 622 insertions(+), 509 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index fdae288..bf8d030 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 3e17025..604e0bb 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ReplicationPeerImpl implements ReplicationPeer {
+
private final Configuration conf;
private final String id;
http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
index ecd888f..cd65f9b 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ServerName;
/**
- * This class is responsible for the parsing logic for a znode representing a
queue.
+ * This class is responsible for the parsing logic for a queue id representing
a queue.
* It will extract the peerId if it's recover
[17/50] [abbrv] hbase git commit: HBASE-19579 Add peer lock test for shell command list_locks
HBASE-19579 Add peer lock test for shell command list_locks
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59d57f82
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59d57f82
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59d57f82
Branch: refs/heads/HBASE-19397-branch-2
Commit: 59d57f82b40029700c996f2acf21196fa222ab78
Parents: b75cffe
Author: Guanghao Zhang
Authored: Sat Dec 23 21:04:27 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../src/main/protobuf/LockService.proto | 1 +
.../src/test/ruby/shell/list_locks_test.rb | 19 +++
2 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/59d57f82/hbase-protocol-shaded/src/main/protobuf/LockService.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
index b8d180c..0675070 100644
--- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
@@ -77,6 +77,7 @@ enum LockedResourceType {
NAMESPACE = 2;
TABLE = 3;
REGION = 4;
+ PEER = 5;
}
message LockedResource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/59d57f82/hbase-shell/src/test/ruby/shell/list_locks_test.rb
--
diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index f465a6b..ef1c0ce 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -67,6 +67,25 @@ module Hbase
proc_id)
end
+define_test 'list peer locks' do
+ lock = create_exclusive_lock(0)
+ peer_id = '1'
+
+ @scheduler.waitPeerExclusiveLock(lock, peer_id)
+ output = capture_stdout { @list_locks.command }
+ @scheduler.wakePeerExclusiveLock(lock, peer_id)
+
+ assert_equal(
+"PEER(1)\n" \
+"Lock type: EXCLUSIVE, procedure: {" \
+
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
+ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\",
\"state\"=>\"RUNNABLE\", " \
+ "\"lastUpdate\"=>\"0\", " \
+ "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\",
\"description\"=>\"description\"}]" \
+"}\n\n",
+output)
+end
+
define_test 'list server locks' do
lock = create_exclusive_lock(0)
[16/50] [abbrv] hbase git commit: HBASE-19216 Implement a general framework to execute remote procedure on RS
HBASE-19216 Implement a general framework to execute remote procedure on RS
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/74f9e73a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/74f9e73a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/74f9e73a
Branch: refs/heads/HBASE-19397-branch-2
Commit: 74f9e73a7a72a75e1aa96641e1b5fe0e0c2117c8
Parents: 309f336
Author: zhangduo
Authored: Fri Dec 15 21:06:44 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../hbase/procedure2/LockedResourceType.java| 4 +-
.../procedure2/RemoteProcedureDispatcher.java | 23 ++-
.../src/main/protobuf/Admin.proto | 9 +-
.../src/main/protobuf/MasterProcedure.proto | 30 +++
.../src/main/protobuf/RegionServerStatus.proto | 15 ++
.../apache/hadoop/hbase/executor/EventType.java | 26 ++-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 30 ++-
.../hadoop/hbase/master/MasterRpcServices.java | 13 ++
.../assignment/RegionTransitionProcedure.java | 18 +-
.../procedure/MasterProcedureScheduler.java | 113 ++-
.../procedure/PeerProcedureInterface.java | 34
.../hbase/master/procedure/PeerQueue.java | 54 +
.../master/procedure/RSProcedureDispatcher.java | 101 ++
.../hbase/master/procedure/SchemaLocking.java | 5 +
.../master/replication/ModifyPeerProcedure.java | 127
.../master/replication/RefreshPeerCallable.java | 67 +++
.../replication/RefreshPeerProcedure.java | 197 +++
.../hbase/procedure2/RSProcedureCallable.java | 43
.../hbase/regionserver/HRegionServer.java | 61 ++
.../hbase/regionserver/RSRpcServices.java | 58 --
.../handler/RSProcedureHandler.java | 51 +
.../assignment/TestAssignmentManager.java | 20 +-
.../replication/DummyModifyPeerProcedure.java | 41
.../TestDummyModifyPeerProcedure.java | 80
.../security/access/TestAccessController.java | 1 +
26 files changed, 1109 insertions(+), 115 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/74f9e73a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index c5fe62b..dc9b5d4 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,5 +22,5 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public enum LockedResourceType {
- SERVER, NAMESPACE, TABLE, REGION
+ SERVER, NAMESPACE, TABLE, REGION, PEER
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/74f9e73a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 6238e10..7e3dde6 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -226,13 +226,30 @@ public abstract class RemoteProcedureDispatcher
- * @param
*/
public interface RemoteProcedure {
+/**
+ * For building the remote operation.
+ */
RemoteOperation remoteCallBuild(TEnv env, TRemote remote);
-void remoteCallCompleted(TEnv env, TRemote remote, RemoteOperation
response);
+
+/**
+ * Called when the executeProcedure call is failed.
+ */
void remoteCallFailed(TEnv env, TRemote remote, IOException exception);
+
+/**
+ * Called when RS tells the remote procedure is succeeded through the
+ * {@code reportProcedureDone} method.
+ */
+void remoteOperationCompleted(TEnv env);
+
+/**
+ * Called when RS tells the remote procedure is failed through the {@code
reportProcedureDone}
+ * method.
+ * @param error the error message
+ */
+void remoteOperationFailed(TEnv env, String error);
[11/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
deleted file mode 100644
index b6f8784..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Manages and performs all replication admin operations.
- *
- * Used to add/remove a replication peer.
- */
-@InterfaceAudience.Private
-public class ReplicationManager {
- private final ReplicationQueuesClient replicationQueuesClient;
- private final ReplicationPeers replicationPeers;
-
- public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable
abortable)
- throws IOException {
-try {
- this.replicationQueuesClient = ReplicationFactory
- .getReplicationQueuesClient(new
ReplicationQueuesClientArguments(conf, abortable, zkw));
- this.replicationQueuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf,
-this.replicationQueuesClient, abortable);
- this.replicationPeers.init();
-} catch (Exception e) {
- throw new IOException("Failed to construct ReplicationManager", e);
-}
- }
-
- public void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig, boolean enabled)
- throws ReplicationException {
-checkPeerConfig(peerConfig);
-replicationPeers.registerPeer(peerId, peerConfig, enabled);
-replicationPeers.peerConnected(peerId);
- }
-
- public void removeReplicationPeer(String peerId) throws ReplicationException
{
-replicationPeers.peerDisconnected(peerId);
-replicationPeers.unregisterPeer(peerId);
- }
-
- public void enableReplicationPeer(String peerId) throws ReplicationException
{
-this.replicationPeers.enablePeer(peerId);
- }
-
- public void disableReplicationPeer(String peerId) throws
ReplicationException {
-this.replicationPeers.disablePeer(peerId);
- }
-
- public ReplicationPeerConfig getPeerConfig(String peerId)
- throws ReplicationException, ReplicationPeerNotFoundException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(peerId);
-if (peerConfig == null) {
- throw new ReplicationPeerNotFoundException(peerId);
-}
-return peerConfig;
- }
-
- public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException, IOException {
-checkPeerConfig(peerConfig);
-this.replicationPeers.updatePeerConfig(peerId, peerConfig);
- }
-
- public List listReplicationPeers(Pattern pattern)
- throws ReplicationException {
-List peers = new ArrayList<>();
-List peerIds = replicationPeers.getAllPeerIds();
-for (String peerId : peer
[07/50] [abbrv] hbase git commit: HBASE-20065 Revisit the timestamp usage in MetaTableAccessor
HBASE-20065 Revisit the timestamp usage in MetaTableAccessor Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d5065bc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d5065bc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d5065bc Branch: refs/heads/HBASE-19397-branch-2 Commit: 2d5065bc7ef832ab84b11173e8b65d913de20bba Parents: 675f76b Author: zhangduo Authored: Sun Feb 25 09:13:34 2018 +0800 Committer: zhangduo Committed: Sun Feb 25 09:14:09 2018 +0800 -- .../apache/hadoop/hbase/MetaTableAccessor.java | 453 --- .../favored/FavoredNodeAssignmentHelper.java| 18 +- .../master/assignment/AssignmentManager.java| 4 - .../master/assignment/RegionStateStore.java | 54 +-- .../hadoop/hbase/regionserver/HRegion.java | 6 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 13 +- .../hadoop/hbase/util/HBaseFsckRepair.java | 4 +- .../hadoop/hbase/TestMetaTableAccessor.java | 54 +-- .../master/assignment/MockMasterServices.java | 8 - .../regionserver/TestGetClosestAtOrBefore.java | 109 ++--- 10 files changed, 301 insertions(+), 422 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5065bc/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 7d00f92..803f183 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -75,58 +75,65 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** - * Read/write operations on region and assignment information store in - * hbase:meta. + * + * Read/write operations on region and assignment information store in hbase:meta. + * + * + * Some of the methods of this class take ZooKeeperWatcher as a param. The only reason for this is + * because when used on client-side (like from HBaseAdmin), we want to use short-living connection + * (opened before each operation, closed right after), while when used on HM or HRS (like in + * AssignmentManager) we want permanent connection. + * + * + * HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the same table + * range (table, startKey, endKey). For every range, there will be at least one HRI defined which is + * called default replica. + * + * + * Meta layout (as of 0.98 + HBASE-10070) is like: * - * Some of the methods of this class take ZooKeeperWatcher as a param. The only reason - * for this is because when used on client-side (like from HBaseAdmin), we want to use - * short-living connection (opened before each operation, closed right after), while - * when used on HM or HRS (like in AssignmentManager) we want permanent connection. + * + * For each table there is single row in column family 'table' formatted: + *including namespace and columns are: + * table: state => contains table state + * + * For each table range, there is a single row, formatted like: + * , , , . + * This row corresponds to the regionName of the default region replica. + * Columns are: + * info:regioninfo => contains serialized HRI for the default region replica + * info:server => contains hostname:port (in string form) for the server hosting + *the default regionInfo replica + * info:server_ => contains hostname:port (in string form) for the server hosting + * the regionInfo replica with replicaId + * info:serverstartcode=> contains server start code (in binary long form) for the server + *hosting the default regionInfo replica + * info:serverstartcode_ => contains server start code (in binary long form) for + * the server hosting the regionInfo replica with + * replicaId + * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time + * the server opened the region with default replicaId + * info:seqnumDuringOpen_ => contains seqNum (in binary long form) for the region + * at the time the server opened the region with + * replicaId + * info:splitA => contains a serialized HRI for the first daughter region if the
[01/50] [abbrv] hbase git commit: HBASE-20048 Revert serial replication feature [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19397-branch-2 9cd0f7014 -> 732d22793 (forced update)
http://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java
deleted file mode 100644
index 88513ea..000
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HTestConst;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Category({ ReplicationTests.class, LargeTests.class })
-public class TestSerialReplication {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestSerialReplication.class);
-
- private static final Logger LOG =
LoggerFactory.getLogger(TestSerialReplication.class);
-
- private static Configuration conf1;
- private static Configuration conf2;
-
- private static HBaseTestingUtility utility1;
- private static HBaseTestingUtility utility2;
-
- private static final byte[] famName = Bytes.toBytes("f");
- private static final byte[] VALUE = Bytes.toBytes("v");
- private static final byte[] ROW = Bytes.toBytes("r");
- private static final byte[][] ROWS = HTestConst.makeNAscii(ROW, 100);
-
- @Rule
- public TestName name = new TestName();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-conf1 = HBaseConfiguration.create();
-conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
-// smaller block size and capacity to trigger more operations
-// and test them
-conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
-conf1.setInt("replication.source.size.capacity", 1024);
-conf1.setLong("replication.source.sleepforretries", 100);
-conf1.setInt("hbase.regionserver.maxlogs", 10);
-conf1.setLong("hbase.master.logcleaner.ttl", 10);
-conf1.setBoolean("dfs.support.append", true);
-conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
-conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KE
[33/50] [abbrv] hbase git commit: HBASE-19623 Create replication endpoint asynchronously when adding a replication source
HBASE-19623 Create replication endpoint asynchronously when adding a
replication source
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a762b60
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a762b60
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a762b60
Branch: refs/heads/HBASE-19397-branch-2
Commit: 8a762b6047f5b9483c955544a4f8f139e5982e60
Parents: 7091095
Author: zhangduo
Authored: Tue Jan 2 13:25:58 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:58 2018 +0800
--
.../hbase/replication/ReplicationPeer.java | 8 ++
.../hbase/replication/ReplicationPeers.java | 18 +--
.../replication/ZKReplicationPeerStorage.java | 7 +-
.../replication/TestReplicationStateBasic.java | 20 +---
.../TestZKReplicationPeerStorage.java | 14 +--
.../HBaseInterClusterReplicationEndpoint.java | 17 ++-
.../RecoveredReplicationSource.java | 13 +--
.../regionserver/ReplicationSource.java | 110 +++
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 47 +---
.../client/TestAsyncReplicationAdminApi.java| 2 -
.../replication/TestReplicationAdmin.java | 2 -
.../replication/ReplicationSourceDummy.java | 7 +-
.../replication/TestReplicationSource.java | 5 +-
.../TestReplicationSourceManager.java | 8 +-
15 files changed, 116 insertions(+), 170 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 4846018..2da3cce 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,14 @@ public interface ReplicationPeer {
PeerState getPeerState();
/**
+ * Test whether the peer is enabled.
+ * @return {@code true} if enabled, otherwise {@code false}.
+ */
+ default boolean isPeerEnabled() {
+return getPeerState() == PeerState.ENABLED;
+ }
+
+ /**
* Get the peer config object
* @return the ReplicationPeerConfig for this peer
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 422801b..45940a5 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -86,21 +87,6 @@ public class ReplicationPeers {
}
/**
- * Get the peer state for the specified connected remote slave cluster. The
value might be read
- * from cache, so it is recommended to use {@link #peerStorage } to read
storage directly if
- * reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- public boolean isPeerEnabled(String peerId) {
-ReplicationPeer replicationPeer = this.peerCache.get(peerId);
-if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + peerId + " is not
cached");
-}
-return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- /**
* Returns the ReplicationPeerImpl for the specified cached peer. This
ReplicationPeer will
* continue to track changes to the Peer's state and config. This method
returns null if no peer
* has been cached with the given peerId.
@@ -117,7 +103,7 @@ public class ReplicationPeers {
* @return a Set of Strings for peerIds
*/
public Set getAllPeerIds() {
-return peerCache.keySet();
+return Collections.unmodifiableSet(peerCache.keySet());
}
public static Configuration
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase
[35/50] [abbrv] hbase git commit: HBASE-19544 Add UTs for testing concurrent modifications on replication peer
HBASE-19544 Add UTs for testing concurrent modifications on replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1d7960ca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1d7960ca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1d7960ca
Branch: refs/heads/HBASE-19397-branch-2
Commit: 1d7960cada9f9e0b3a1319ade155c0f4f5d3b8c9
Parents: a75ee53
Author: Guanghao Zhang
Authored: Tue Jan 2 17:07:41 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:56:58 2018 +0800
--
.../replication/TestReplicationAdmin.java | 69
1 file changed, 69 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1d7960ca/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 772a9d6..a753d23 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -58,6 +59,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Unit testing of ReplicationAdmin
@@ -69,6 +72,8 @@ public class TestReplicationAdmin {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestReplicationAdmin.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(TestReplicationAdmin.class);
+
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
@@ -118,6 +123,70 @@ public class TestReplicationAdmin {
}
@Test
+ public void testConcurrentPeerOperations() throws Exception {
+int threadNum = 5;
+AtomicLong successCount = new AtomicLong(0);
+
+// Test concurrent add peer operation
+Thread[] addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent remove peer operation
+successCount.set(0);
+Thread[] removePeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ removePeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.removeReplicationPeer(ID_ONE);
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when remove replication peer", e);
+}
+ });
+ removePeers[i].start();
+}
+for (Thread removePeer : removePeers) {
+ removePeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent add peer operation again
+successCount.set(0);
+addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+ }
+
+ @Test
public void testAddInvalidPeer() {
ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
builder.setClusterKey(KEY_ONE);
[10/50] [abbrv] hbase git commit: HBASE-20019 Document the ColumnValueFilter
HBASE-20019 Document the ColumnValueFilter
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/309f3360
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/309f3360
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/309f3360
Branch: refs/heads/HBASE-19397-branch-2
Commit: 309f3360bf3f45c7ea572a09eddada02cf6b93b3
Parents: 10270e3
Author: Reid Chan
Authored: Mon Feb 26 11:31:08 2018 +0800
Committer: Chia-Ping Tsai
Committed: Mon Feb 26 15:11:01 2018 +0800
--
src/main/asciidoc/_chapters/architecture.adoc | 35 ++
1 file changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/309f3360/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc
b/src/main/asciidoc/_chapters/architecture.adoc
index 5ba81c1..b29244c 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -321,6 +321,41 @@ SingleColumnValueFilter filter = new
SingleColumnValueFilter(
scan.setFilter(filter);
+[[client.filter.cv.cvf]]
+ ColumnValueFilter
+
+Introduced in HBase-2.0.0 version as a complementation of
SingleColumnValueFilter, ColumnValueFilter
+gets matched cell only, while SingleColumnValueFilter gets the entire row
+(has other columns and values) to which the matched cell belongs. Parameters
of constructor of
+ColumnValueFilter are the same as SingleColumnValueFilter.
+[source,java]
+
+ColumnValueFilter filter = new ColumnValueFilter(
+ cf,
+ column,
+ CompareOperaor.EQUAL,
+ Bytes.toBytes("my value")
+ );
+scan.setFilter(filter);
+
+
+Note. For simple query like "equals to a family:qualifier:value", we highly
recommend to use the
+following way instead of using SingleColumnValueFilter or ColumnValueFilter:
+[source,java]
+
+Scan scan = new Scan();
+scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qualifier"));
+ValueFilter vf = new ValueFilter(CompareOperator.EQUAL,
+ new BinaryComparator(Bytes.toBytes("value")));
+scan.setFilter(vf);
+...
+
+This scan will restrict to the specified column 'family:qualifier', avoiding
scan unrelated
+families and columns, which has better performance, and `ValueFilter` is the
condition used to do
+the value filtering.
+
+But if query is much more complicated beyond this book, then please make your
good choice case by case.
+
[[client.filter.cvp]]
=== Column Value Comparators
[31/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index b8d80d2..5b7bab8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -33,17 +33,20 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -56,7 +59,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -91,7 +94,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
private final List sources;
// List of all the sources we got from died RSs
private final List oldsources;
- private final ReplicationQueues replicationQueues;
+ private final ReplicationQueueStorage queueStorage;
private final ReplicationTracker replicationTracker;
private final ReplicationPeers replicationPeers;
// UUID for this cluster
@@ -124,7 +127,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
/**
* Creates a replication manager and sets the watch on all the other
registered region servers
- * @param replicationQueues the interface for manipulating replication queues
+ * @param queueStorage the interface for manipulating replication queues
* @param replicationPeers
* @param replicationTracker
* @param conf the configuration to use
@@ -134,14 +137,14 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param oldLogDir the directory where old logs are archived
* @param clusterId
*/
- public ReplicationSourceManager(ReplicationQueues replicationQueues,
+ public ReplicationSourceManager(ReplicationQueueStorage queueStorage,
ReplicationPeers replicationPeers, ReplicationTracker
replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID
clusterId,
WALFileLengthProvider walFileLengthProvider) throws IOException {
//CopyOnWriteArrayList is thread-safe.
//Generally, reading is more than modifying.
this.sources = new CopyOnWriteArrayList<>();
-this.replicationQueues = replicationQueues;
+this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.replicationTracker = replicationTracker;
this.server = server;
@@ -174,6 +177,19 @@ public class ReplicationSourceManager implements
ReplicationListener {
HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
}
+ @FunctionalInterface
+ private interface ReplicationQueueOperation {
+void exec() throws ReplicationException;
+ }
+
+ private void abortWhenFail(ReplicationQueueOperation op) {
+try {
+ op.exec();
+} catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+}
+ }
+
/**
* Provide the id of the peer and a log key and this method will figure which
* wal it belongs to and will log, for this region server, the current
@@ -185,12 +201,13 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param queueRecovered indicates if this queue comes from another region
server
* @param holdLogInZK if true then the log is retained in
[21/50] [abbrv] hbase git commit: HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/751a6eb8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/751a6eb8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/751a6eb8
Branch: refs/heads/HBASE-19397-branch-2
Commit: 751a6eb8ff88ec1ccc4df027181a8c01323c640c
Parents: 59d57f8
Author: Guanghao Zhang
Authored: Tue Dec 26 11:39:34 2017 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:52:10 2018 +0800
--
.../replication/VerifyReplication.java | 5 -
.../hbase/replication/ReplicationPeer.java | 42 ++--
.../hbase/replication/ReplicationPeerImpl.java | 169 ++
.../replication/ReplicationPeerZKImpl.java | 233 ---
.../hbase/replication/ReplicationPeers.java | 4 +-
.../replication/ReplicationPeersZKImpl.java | 23 +-
.../replication/TestReplicationStateBasic.java | 7 +-
.../regionserver/PeerProcedureHandlerImpl.java | 29 +--
8 files changed, 216 insertions(+), 296 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/751a6eb8/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 9065f4e..09d4b4b 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
@@ -333,7 +332,6 @@ public class VerifyReplication extends Configured
implements Tool {
private static Pair
getPeerQuorumConfig(
final Configuration conf, String peerId) throws IOException {
ZKWatcher localZKW = null;
-ReplicationPeerZKImpl peer = null;
try {
localZKW = new ZKWatcher(conf, "VerifyReplication",
new Abortable() {
@@ -354,9 +352,6 @@ public class VerifyReplication extends Configured
implements Tool {
throw new IOException(
"An error occurred while trying to connect to the remove peer
cluster", e);
} finally {
- if (peer != null) {
-peer.close();
- }
if (localZKW != null) {
localZKW.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/751a6eb8/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index b66d76d..4846018 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* ReplicationPeer manages enabled / disabled state for the peer.
*/
@@ -49,65 +48,52 @@ public interface ReplicationPeer {
String getId();
/**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig();
-
- /**
- * Get the peer config object. if loadFromBackingStore is true, it will load
from backing store
- * directly and update its load peer config. otherwise, just return the
local cached peer config.
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
- throws ReplicationException;
-
- /**
* Returns the state of the peer by reading local cache.
* @return the enabled state
*/
PeerState getPeerState();
/**
- * Returns the state of peer, if loadFromBackingStore is true, it will load
from backing store
- * directly and update its local peer state. otherwise, just return t
[18/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 6e27a21..d8f9625 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,13 +21,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,17 +48,18 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
@@ -303,57 +304,53 @@ public class DumpReplicationQueues extends Configured
implements Tool {
}
public String dumpQueues(ClusterConnection connection, ZKWatcher zkw,
Set peerIds,
- boolean hdfs) throws Exception {
-ReplicationQueuesClient queuesClient;
+ boolean hdfs) throws Exception {
+ReplicationQueueStorage queueStorage;
ReplicationPeers replicationPeers;
ReplicationQueues replicationQueues;
ReplicationTracker replicationTracker;
-ReplicationQueuesClientArguments replicationArgs =
-new ReplicationQueuesClientArguments(getConf(), new
WarnOnlyAbortable(), zkw);
+ReplicationQueuesArguments replicationArgs =
+new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(),
zkw);
StringBuilder sb = new StringBuilder();
-queuesClient =
ReplicationFactory.getReplicationQueuesClient(replicationArgs);
-queuesClient.init();
+queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw,
getConf());
replicationQueues =
ReplicationFactory.getReplicationQueues(replicationArgs);
-replicationPeers = ReplicationFactory.getReplicationPeers(zkw, getConf(),
queuesClient, connection);
+replicationPeers =
+ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage,
connection);
replicationTracker = ReplicationFactory.getReplicationTracker(zkw,
replicationPeers, getConf(),
new WarnOnlyAbortable(), new WarnOnlyStoppable());
-List liveRegionServers =
replicationTracker.getListOfRegionServers();
+Set liveRegionServers = new
HashSet<>(replicationTracker.getListOfRegionServers());
// Loops each peer on each RS and dumps the queues
-try {
- List regionservers = queuesClient.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
-return sb.toString();
+List regionservers = queueStorage.getListOfReplicators();
+if (regionservers == null || regionservers.isEmpty()) {
+ return sb.toString();
+}
+for (ServerName regionserver : regionservers) {
+ List queueIds = queueStorage.getAllQueues(regionserver);
+ replicationQueues.init(regionserver.getServerName());
+ if (!liveRegionServers.contains(regionserver.getServerName())) {
+deadRegionServers.add(regionserver.getServerName());
}
- for (String regionserver : regionservers) {
-List queueIds = queuesClient.getAllQueues(regionserver);
-replicationQueues.init(regionserver);
-if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver);
-}
-for (String
hbase git commit: HBASE-20083 Fix findbugs error for ReplicationSyncUp
Repository: hbase
Updated Branches:
refs/heads/master 8c74d177f -> 2beda62a1
HBASE-20083 Fix findbugs error for ReplicationSyncUp
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2beda62a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2beda62a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2beda62a
Branch: refs/heads/master
Commit: 2beda62a10f0828eb10cec28b0ba53246cd0b671
Parents: 8c74d17
Author: zhangduo
Authored: Mon Feb 26 16:37:58 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 22:13:13 2018 +0800
--
.../regionserver/ReplicationSyncUp.java | 59 ++--
.../replication/TestReplicationSyncUpTool.java | 6 +-
2 files changed, 19 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2beda62a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 283eb96..c2862de 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -49,35 +49,18 @@ import org.apache.hadoop.util.ToolRunner;
*/
public class ReplicationSyncUp extends Configured implements Tool {
- private static Configuration conf;
-
private static final long SLEEP_TIME = 1;
- // although the tool is designed to be run on command line
- // this api is provided for executing the tool through another app
- public static void setConfigure(Configuration config) {
-conf = config;
- }
-
/**
* Main program
- * @param args
- * @throws Exception
*/
public static void main(String[] args) throws Exception {
-if (conf == null) conf = HBaseConfiguration.create();
-int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args);
+int ret = ToolRunner.run(HBaseConfiguration.create(), new
ReplicationSyncUp(), args);
System.exit(ret);
}
@Override
public int run(String[] args) throws Exception {
-Replication replication;
-ReplicationSourceManager manager;
-FileSystem fs;
-Path oldLogDir, logDir, walRootDir;
-ZKWatcher zkw;
-
Abortable abortable = new Abortable() {
@Override
public void abort(String why, Throwable e) {
@@ -88,23 +71,19 @@ public class ReplicationSyncUp extends Configured
implements Tool {
return false;
}
};
-
-zkw =
-new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable,
-true);
-
-walRootDir = FSUtils.getWALRootDir(conf);
-fs = FSUtils.getWALFileSystem(conf);
-oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
-logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
-
-System.out.println("Start Replication Server start");
-replication = new Replication();
-replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null);
-manager = replication.getReplicationManager();
-manager.init().get();
-
-try {
+Configuration conf = getConf();
+try (ZKWatcher zkw =
+ new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable, true)) {
+ Path walRootDir = FSUtils.getWALRootDir(conf);
+ FileSystem fs = FSUtils.getWALFileSystem(conf);
+ Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
+
+ System.out.println("Start Replication Server start");
+ Replication replication = new Replication();
+ replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir,
null);
+ ReplicationSourceManager manager = replication.getReplicationManager();
+ manager.init().get();
while (manager.activeFailoverTaskCount() > 0) {
Thread.sleep(SLEEP_TIME);
}
@@ -114,15 +93,12 @@ public class ReplicationSyncUp extends Configured
implements Tool {
manager.join();
} catch (InterruptedException e) {
System.err.println("didn't wait long enough:" + e);
- return (-1);
-} finally {
- zkw.close();
+ return -1;
}
-
return 0;
}
- static class DummyServer implements Server {
+ class DummyServer implements Server {
String hostname;
ZKWatcher zkw;
@@ -138,7 +114,7 @@ public class ReplicationSyncUp extends Configured
implements Tool {
@Override
public Configuration getConfiguration() {
- return conf;
hbase git commit: HBASE-20083 Fix findbugs error for ReplicationSyncUp
Repository: hbase
Updated Branches:
refs/heads/branch-2 309f3360b -> ef02762dd
HBASE-20083 Fix findbugs error for ReplicationSyncUp
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ef02762d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ef02762d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ef02762d
Branch: refs/heads/branch-2
Commit: ef02762dd8ac314c4d027deaf9cc5a913d68b791
Parents: 309f336
Author: zhangduo
Authored: Mon Feb 26 16:37:58 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 22:16:37 2018 +0800
--
.../regionserver/ReplicationSyncUp.java | 79 +++-
.../replication/TestReplicationSyncUpTool.java | 6 +-
2 files changed, 27 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ef02762d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 43c7bce..f7ea78f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -37,51 +36,31 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * In a scenario of Replication based Disaster/Recovery, when hbase
- * Master-Cluster crashes, this tool is used to sync-up the delta from Master
to
- * Slave using the info from ZooKeeper. The tool will run on Master-Cluser, and
- * assume ZK, Filesystem and NetWork still available after hbase crashes
- *
+ * In a scenario of Replication based Disaster/Recovery, when hbase
Master-Cluster crashes, this
+ * tool is used to sync-up the delta from Master to Slave using the info from
ZooKeeper. The tool
+ * will run on Master-Cluser, and assume ZK, Filesystem and NetWork still
available after hbase
+ * crashes
+ *
+ *
* hbase org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp
+ *
*/
-
public class ReplicationSyncUp extends Configured implements Tool {
- private static final Logger LOG =
LoggerFactory.getLogger(ReplicationSyncUp.class.getName());
-
- private static Configuration conf;
-
private static final long SLEEP_TIME = 1;
- // although the tool is designed to be run on command line
- // this api is provided for executing the tool through another app
- public static void setConfigure(Configuration config) {
-conf = config;
- }
-
/**
* Main program
- * @param args
- * @throws Exception
*/
public static void main(String[] args) throws Exception {
-if (conf == null) conf = HBaseConfiguration.create();
-int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args);
+int ret = ToolRunner.run(HBaseConfiguration.create(), new
ReplicationSyncUp(), args);
System.exit(ret);
}
@Override
public int run(String[] args) throws Exception {
-Replication replication;
-ReplicationSourceManager manager;
-FileSystem fs;
-Path oldLogDir, logDir, walRootDir;
-ZKWatcher zkw;
-
Abortable abortable = new Abortable() {
@Override
public void abort(String why, Throwable e) {
@@ -92,23 +71,19 @@ public class ReplicationSyncUp extends Configured
implements Tool {
return false;
}
};
-
-zkw =
-new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable,
-true);
-
-walRootDir = FSUtils.getWALRootDir(conf);
-fs = FSUtils.getWALFileSystem(conf);
-oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
-logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
-
-System.out.println("Start Replication Server start");
-replication = new Replication();
-replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null);
-manager = replication.getReplicationManager();
-manager.init();
-
-try {
+Configuration conf = getConf();
+try (ZKWatcher zkw =
+ new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable, true)) {
+ Path walRootDir =
hbase git commit: HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods
Repository: hbase
Updated Branches:
refs/heads/master 2beda62a1 -> a29b3caf4
HBASE-19974 Fix decommissioned servers cannot be removed by
remove_servers_rsgroup methods
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a29b3caf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a29b3caf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a29b3caf
Branch: refs/heads/master
Commit: a29b3caf4dbc7b8833474ef5da5438f7f6907e00
Parents: 2beda62
Author: haxiaolin
Authored: Mon Feb 26 14:25:01 2018 +0800
Committer: tedyu
Committed: Mon Feb 26 07:30:56 2018 -0800
--
.../hbase/rsgroup/RSGroupAdminServer.java | 8 +++-
.../hadoop/hbase/rsgroup/TestRSGroups.java | 7 ++-
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 46
3 files changed, 40 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
index aba57fe..094fc1d 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
@@ -675,8 +675,12 @@ public class RSGroupAdminServer implements RSGroupAdmin {
private void checkForDeadOrOnlineServers(Set servers) throws
ConstraintException {
// This uglyness is because we only have Address, not ServerName.
Set onlineServers = new HashSet<>();
-for(ServerName server:
master.getServerManager().getOnlineServers().keySet()) {
- onlineServers.add(server.getAddress());
+List drainingServers =
master.getServerManager().getDrainingServersList();
+for (ServerName server :
master.getServerManager().getOnlineServers().keySet()) {
+ // Only online but not decommissioned servers are really online
+ if (!drainingServers.contains(server)) {
+onlineServers.add(server.getAddress());
+ }
}
Set deadServers = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
index 9116f3b..610278a 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.net.Address;
@@ -66,7 +65,6 @@ public class TestRSGroups extends TestRSGroupsBase {
HBaseClassTestRule.forClass(TestRSGroups.class);
protected static final Logger LOG =
LoggerFactory.getLogger(TestRSGroups.class);
- private static HMaster master;
private static boolean INIT = false;
private static RSGroupAdminEndpoint rsGroupAdminEndpoint;
@@ -126,6 +124,11 @@ public class TestRSGroups extends TestRSGroupsBase {
deleteNamespaceIfNecessary();
deleteGroups();
+for(ServerName sn : admin.listDecommissionedRegionServers()){
+ admin.recommissionRegionServer(sn, null);
+}
+assertTrue(admin.listDecommissionedRegionServers().isEmpty());
+
int missing = NUM_SLAVES_BASE - getNumServers();
LOG.info("Restoring servers: "+missing);
for(int i=0; ihttp://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index c3f7eef..76bcd20 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -25,8 +25,10 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.secur
hbase git commit: HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods
Repository: hbase
Updated Branches:
refs/heads/branch-2 ef02762dd -> 0bf33c802
HBASE-19974 Fix decommissioned servers cannot be removed by
remove_servers_rsgroup methods
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0bf33c80
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0bf33c80
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0bf33c80
Branch: refs/heads/branch-2
Commit: 0bf33c802d7ad194f39cb80e86a0255b2155f455
Parents: ef02762
Author: haxiaolin
Authored: Mon Feb 26 14:25:01 2018 +0800
Committer: tedyu
Committed: Mon Feb 26 07:31:58 2018 -0800
--
.../hbase/rsgroup/RSGroupAdminServer.java | 8 +++-
.../hadoop/hbase/rsgroup/TestRSGroups.java | 7 ++-
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 46
3 files changed, 40 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
index 57d5412..f7e7731 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
@@ -675,8 +675,12 @@ public class RSGroupAdminServer implements RSGroupAdmin {
private void checkForDeadOrOnlineServers(Set servers) throws
ConstraintException {
// This uglyness is because we only have Address, not ServerName.
Set onlineServers = new HashSet<>();
-for(ServerName server:
master.getServerManager().getOnlineServers().keySet()) {
- onlineServers.add(server.getAddress());
+List drainingServers =
master.getServerManager().getDrainingServersList();
+for (ServerName server :
master.getServerManager().getOnlineServers().keySet()) {
+ // Only online but not decommissioned servers are really online
+ if (!drainingServers.contains(server)) {
+onlineServers.add(server.getAddress());
+ }
}
Set deadServers = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
index 9116f3b..610278a 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.net.Address;
@@ -66,7 +65,6 @@ public class TestRSGroups extends TestRSGroupsBase {
HBaseClassTestRule.forClass(TestRSGroups.class);
protected static final Logger LOG =
LoggerFactory.getLogger(TestRSGroups.class);
- private static HMaster master;
private static boolean INIT = false;
private static RSGroupAdminEndpoint rsGroupAdminEndpoint;
@@ -126,6 +124,11 @@ public class TestRSGroups extends TestRSGroupsBase {
deleteNamespaceIfNecessary();
deleteGroups();
+for(ServerName sn : admin.listDecommissionedRegionServers()){
+ admin.recommissionRegionServer(sn, null);
+}
+assertTrue(admin.listDecommissionedRegionServers().isEmpty());
+
int missing = NUM_SLAVES_BASE - getNumServers();
LOG.info("Restoring servers: "+missing);
for(int i=0; ihttp://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index cae9da3..fdc29b3 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -25,8 +25,10 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.s
hbase git commit: HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Repository: hbase
Updated Branches:
refs/heads/branch-2 0bf33c802 -> 372c68fca
HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/372c68fc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/372c68fc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/372c68fc
Branch: refs/heads/branch-2
Commit: 372c68fca28383d08128ce1c376492706b1d66f3
Parents: 0bf33c8
Author: Vasudevan
Authored: Mon Feb 26 22:05:12 2018 +0530
Committer: Vasudevan
Committed: Mon Feb 26 22:06:28 2018 +0530
--
...estAvoidCellReferencesIntoShippedBlocks.java | 30 +++-
1 file changed, 17 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/372c68fc/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index 0e12ad6..d22772a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -400,23 +400,27 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
scanner = table.getScanner(s1);
int count = Iterables.size(scanner);
assertEquals("Count the rows", 2, count);
- iterator = cache.iterator();
- List newCacheList = new ArrayList<>();
- while (iterator.hasNext()) {
-CachedBlock next = iterator.next();
-BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(),
next.getOffset());
-newCacheList.add(cacheKey);
- }
int newBlockRefCount = 0;
- for (BlockCacheKey key : cacheList) {
-if (newCacheList.contains(key)) {
- newBlockRefCount++;
+ List newCacheList = new ArrayList<>();
+ while (true) {
+newBlockRefCount = 0;
+newCacheList.clear();
+iterator = cache.iterator();
+while (iterator.hasNext()) {
+ CachedBlock next = iterator.next();
+ BlockCacheKey cacheKey = new
BlockCacheKey(next.getFilename(), next.getOffset());
+ newCacheList.add(cacheKey);
+}
+for (BlockCacheKey key : cacheList) {
+ if (newCacheList.contains(key)) {
+newBlockRefCount++;
+ }
+}
+if (newBlockRefCount == 6) {
+ break;
}
}
-
- assertEquals("old blocks should still be found ", 6,
newBlockRefCount);
latch.countDown();
-
} catch (IOException e) {
}
}
hbase git commit: HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Repository: hbase
Updated Branches:
refs/heads/master a29b3caf4 -> 7cfb46432
HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cfb4643
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cfb4643
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cfb4643
Branch: refs/heads/master
Commit: 7cfb46432fbdf9b53592be11efc8a7d79d1a9455
Parents: a29b3ca
Author: Vasudevan
Authored: Mon Feb 26 22:07:42 2018 +0530
Committer: Vasudevan
Committed: Mon Feb 26 22:07:42 2018 +0530
--
...estAvoidCellReferencesIntoShippedBlocks.java | 30 +++-
1 file changed, 17 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cfb4643/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index 0e12ad6..d22772a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -400,23 +400,27 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
scanner = table.getScanner(s1);
int count = Iterables.size(scanner);
assertEquals("Count the rows", 2, count);
- iterator = cache.iterator();
- List newCacheList = new ArrayList<>();
- while (iterator.hasNext()) {
-CachedBlock next = iterator.next();
-BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(),
next.getOffset());
-newCacheList.add(cacheKey);
- }
int newBlockRefCount = 0;
- for (BlockCacheKey key : cacheList) {
-if (newCacheList.contains(key)) {
- newBlockRefCount++;
+ List newCacheList = new ArrayList<>();
+ while (true) {
+newBlockRefCount = 0;
+newCacheList.clear();
+iterator = cache.iterator();
+while (iterator.hasNext()) {
+ CachedBlock next = iterator.next();
+ BlockCacheKey cacheKey = new
BlockCacheKey(next.getFilename(), next.getOffset());
+ newCacheList.add(cacheKey);
+}
+for (BlockCacheKey key : cacheList) {
+ if (newCacheList.contains(key)) {
+newBlockRefCount++;
+ }
+}
+if (newBlockRefCount == 6) {
+ break;
}
}
-
- assertEquals("old blocks should still be found ", 6,
newBlockRefCount);
latch.countDown();
-
} catch (IOException e) {
}
}
[6/6] hbase git commit: HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
HBASE-20016
TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
(cherry picked from commit 08b993997404aad7ec8f66b8c2b2083062c9ef1a)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41cd37cf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41cd37cf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41cd37cf
Branch: refs/heads/branch-1.2
Commit: 41cd37cffc690d056edf1ca10d5cbc07d02373c5
Parents: 4b79555
Author: Chia-Ping Tsai
Authored: Sun Feb 18 21:22:58 2018 +0800
Committer: Sean Busbey
Committed: Mon Feb 26 10:46:57 2018 -0600
--
.../TestCatalogJanitorInMemoryStates.java | 52 +---
1 file changed, 22 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/41cd37cf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
index 6ea7e2f..40396ba 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
@@ -124,7 +124,7 @@ public class TestCatalogJanitorInMemoryStates {
* AM's serverHoldings and
*/
@Test(timeout = 18)
- public void testInMemoryForReplicaParentCleanup() throws IOException,
InterruptedException {
+ public void testInMemoryForReplicaParentCleanup() throws Exception {
final AssignmentManager am =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
final CatalogJanitor janitor =
TEST_UTIL.getHBaseCluster().getMaster().catalogJanitorChore;
@@ -139,35 +139,27 @@ public class TestCatalogJanitorInMemoryStates {
// There are two regions, one for primary, one for the replica.
assertTrue(allRegionLocations.size() == 2);
-HRegionLocation replicaParentRegion, primaryParentRegion;
-if (RegionReplicaUtil.isDefaultReplica(
-allRegionLocations.get(0).getRegionInfo().getReplicaId())) {
- primaryParentRegion = allRegionLocations.get(0);
- replicaParentRegion = allRegionLocations.get(1);
-} else {
- primaryParentRegion = allRegionLocations.get(1);
- replicaParentRegion = allRegionLocations.get(0);
-}
-
-List primaryDaughters =
splitRegion(primaryParentRegion.getRegionInfo(),
-Bytes.toBytes("a"));
-
-// Wait until the replica parent region is offline.
-while
(am.getRegionStates().isRegionOnline(replicaParentRegion.getRegionInfo())) {
- Thread.sleep(100);
-}
-
-assertNotNull("Should have found daughter regions for " +
primaryDaughters, primaryDaughters);
-
-// check that primary parent region is not in AM's serverHoldings
-assertFalse("Primary Parent region should have been removed from
RegionState's serverHoldings",
-
am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(),
-primaryParentRegion.getRegionInfo()));
-
-// check that primary parent region is not in AM's serverHoldings
-assertFalse("Primary Parent region should have been removed from
RegionState's serverHoldings",
-
am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(),
-replicaParentRegion.getRegionInfo()));
+final HRegionLocation primaryParentRegion
+ = RegionReplicaUtil.isDefaultReplica(
+ allRegionLocations.get(0).getRegionInfo().getReplicaId()) ?
allRegionLocations.get(0)
+: allRegionLocations.get(1);
+final HRegionLocation replicaParentRegion
+ = RegionReplicaUtil.isDefaultReplica(
+ allRegionLocations.get(0).getRegionInfo().getReplicaId()) ?
allRegionLocations.get(1)
+ : allRegionLocations.get(0);
+
+assertNotNull("Should have found daughter regions for " +
primaryParentRegion,
+ splitRegion(primaryParentRegion.getRegionInfo(), Bytes.toBytes("a")));
+
+TEST_UTIL.waitFor(60 * 1000, new Waiter.Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+return
!am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(),
+ primaryParentRegion.getRegionInfo()) &&
+
!am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(),
+replicaParentRegion.getRegionInfo());
+ }
+});
}
/*
[3/6] hbase git commit: HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
HBASE-20016
TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
(cherry picked from commit 08b993997404aad7ec8f66b8c2b2083062c9ef1a)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f96c356
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f96c356
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f96c356
Branch: refs/heads/branch-1.3
Commit: 5f96c356b8dddaf20f8df3207117400999097127
Parents: 5ed1f41
Author: Chia-Ping Tsai
Authored: Sun Feb 18 21:22:58 2018 +0800
Committer: Sean Busbey
Committed: Mon Feb 26 09:47:31 2018 -0600
--
.../TestCatalogJanitorInMemoryStates.java | 52 +---
1 file changed, 22 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f96c356/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
index 6ea7e2f..40396ba 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
@@ -124,7 +124,7 @@ public class TestCatalogJanitorInMemoryStates {
* AM's serverHoldings and
*/
@Test(timeout = 18)
- public void testInMemoryForReplicaParentCleanup() throws IOException,
InterruptedException {
+ public void testInMemoryForReplicaParentCleanup() throws Exception {
final AssignmentManager am =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
final CatalogJanitor janitor =
TEST_UTIL.getHBaseCluster().getMaster().catalogJanitorChore;
@@ -139,35 +139,27 @@ public class TestCatalogJanitorInMemoryStates {
// There are two regions, one for primary, one for the replica.
assertTrue(allRegionLocations.size() == 2);
-HRegionLocation replicaParentRegion, primaryParentRegion;
-if (RegionReplicaUtil.isDefaultReplica(
-allRegionLocations.get(0).getRegionInfo().getReplicaId())) {
- primaryParentRegion = allRegionLocations.get(0);
- replicaParentRegion = allRegionLocations.get(1);
-} else {
- primaryParentRegion = allRegionLocations.get(1);
- replicaParentRegion = allRegionLocations.get(0);
-}
-
-List primaryDaughters =
splitRegion(primaryParentRegion.getRegionInfo(),
-Bytes.toBytes("a"));
-
-// Wait until the replica parent region is offline.
-while
(am.getRegionStates().isRegionOnline(replicaParentRegion.getRegionInfo())) {
- Thread.sleep(100);
-}
-
-assertNotNull("Should have found daughter regions for " +
primaryDaughters, primaryDaughters);
-
-// check that primary parent region is not in AM's serverHoldings
-assertFalse("Primary Parent region should have been removed from
RegionState's serverHoldings",
-
am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(),
-primaryParentRegion.getRegionInfo()));
-
-// check that primary parent region is not in AM's serverHoldings
-assertFalse("Primary Parent region should have been removed from
RegionState's serverHoldings",
-
am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(),
-replicaParentRegion.getRegionInfo()));
+final HRegionLocation primaryParentRegion
+ = RegionReplicaUtil.isDefaultReplica(
+ allRegionLocations.get(0).getRegionInfo().getReplicaId()) ?
allRegionLocations.get(0)
+: allRegionLocations.get(1);
+final HRegionLocation replicaParentRegion
+ = RegionReplicaUtil.isDefaultReplica(
+ allRegionLocations.get(0).getRegionInfo().getReplicaId()) ?
allRegionLocations.get(1)
+ : allRegionLocations.get(0);
+
+assertNotNull("Should have found daughter regions for " +
primaryParentRegion,
+ splitRegion(primaryParentRegion.getRegionInfo(), Bytes.toBytes("a")));
+
+TEST_UTIL.waitFor(60 * 1000, new Waiter.Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+return
!am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(),
+ primaryParentRegion.getRegionInfo()) &&
+
!am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(),
+replicaParentRegion.getRegionInfo());
+ }
+});
}
/*
[4/6] hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
(cherry picked from commit aaece0ba5e399f248c8255fe509cdb1a862bf299)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/688d4dec
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/688d4dec
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/688d4dec
Branch: refs/heads/branch-1.2
Commit: 688d4dec74d16f7641a347f3dcd7aa4fa686758d
Parents: d2fe2a5
Author: Esteban Gutierrez
Authored: Fri Aug 11 12:56:20 2017 -0500
Committer: Sean Busbey
Committed: Mon Feb 26 10:46:44 2018 -0600
--
.../hadoop/hbase/master/CatalogJanitor.java | 4 +
.../hadoop/hbase/master/RegionStates.java | 6 +
.../hadoop/hbase/master/ServerManager.java | 7 +
.../TestCatalogJanitorInMemoryStates.java | 188 +++
4 files changed, 205 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 48105c8..893b108 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -216,6 +216,8 @@ public class CatalogJanitor extends ScheduledChore {
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionB);
MetaTableAccessor.deleteMergeQualifiers(services.getConnection(),
mergedRegion);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
services.getServerManager().removeRegion(regionA);
services.getServerManager().removeRegion(regionB);
return true;
@@ -350,6 +352,8 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " +
parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ if (services.getAssignmentManager().getRegionStates() != null)
+services.getAssignmentManager().getRegionStates().deleteRegion(parent);
services.getServerManager().removeRegion(parent);
result = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index bfdfecc..cf68a02 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -848,6 +848,12 @@ public class RegionStates {
}
}
+ @VisibleForTesting
+ public boolean isRegionInRegionStates(final HRegionInfo hri) {
+return (getRegionState(hri) != null || isRegionOnline(hri)) ||
isRegionInTransition(hri)
+|| isRegionInState(hri, State.OFFLINE, State.CLOSED);
+ }
+
/**
* Checking if a region was assigned to a server which is not online now.
* If so, we should hold re-assign this region till SSH has split its wals.
http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index a420705..7527af7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1222,6 +1222,13 @@ public class ServerManager {
flushedSequenceIdByRegion.remove(encodedName);
}
+ @VisibleForTesting
+ public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+final byte[] encodedName = hri.getEncodedNameAsBytes();
+return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+|| flushedSequenceIdByRegion.containsKey(encodedName));
+ }
+
/**
* Called by delete table and similar to notify the ServerManager that a
region was removed.
*/
http://git-wip-us.apache.org/repos/a
[2/6] hbase git commit: HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap.
HBASE-18625 Splitting of region with replica, doesn't update region list in
serverHolding. A server crash leads to overlap.
Signed-off-by: ramkrishna.s.vasudevan
(cherry picked from commit 80d183a707c6d0203da1565f6a7ca3edf4bb2291)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ed1f416
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ed1f416
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ed1f416
Branch: refs/heads/branch-1.3
Commit: 5ed1f4166e7f7afb6e5625f5cecd5e045301c15d
Parents: 4b44c90
Author: huaxiang sun
Authored: Wed Jan 3 12:24:05 2018 -0800
Committer: Sean Busbey
Committed: Mon Feb 26 09:47:26 2018 -0600
--
.../hadoop/hbase/master/RegionStates.java | 33 ++-
.../TestCatalogJanitorInMemoryStates.java | 91 +---
2 files changed, 112 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ed1f416/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 406f5e6..3d0c840 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -555,6 +555,19 @@ public class RegionStates {
}
/**
+ * Used in some unit tests
+ */
+ @VisibleForTesting
+ synchronized boolean existsInServerHoldings(final ServerName serverName,
+ final HRegionInfo hri) {
+Set oldRegions = serverHoldings.get(serverName);
+if (oldRegions != null) {
+ return oldRegions.contains(hri);
+}
+return false;
+ }
+
+ /**
* A dead server's wals have been split so that all the regions
* used to be open on it can be safely assigned now. Mark them assignable.
*/
@@ -623,8 +636,26 @@ public class RegionStates {
deleteRegion(hri);
return;
}
+
+/*
+ * One tricky case, if region here is a replica region and its parent is at
+ * SPLIT state, its newState should be same as its parent, not OFFLINE.
+ */
State newState =
- expectedState == null ? State.OFFLINE : expectedState;
+expectedState == null ? State.OFFLINE : expectedState;
+
+if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) {
+ RegionState primateState = getRegionState(
+ RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
+ if ((primateState != null) && (primateState.getState() == State.SPLIT)) {
+if (LOG.isDebugEnabled()) {
+ LOG.debug("Update region " + hri + "to SPLIT, from primary region " +
+ RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
+}
+newState = State.SPLIT;
+ }
+}
+
updateRegionState(hri, newState);
String encodedName = hri.getEncodedName();
synchronized (this) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ed1f416/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
index d2bed9b..6ea7e2f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
@@ -18,21 +18,20 @@
*/
package org.apache.hadoop.hbase.master;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.master.AssignmentManager;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.AfterClass;
@@ -44,12 +43,6 @@ import org.junit.experimental.categories.Cate
[1/6] hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 d2fe2a5a9 -> 41cd37cff
refs/heads/branch-1.3 1a36db146 -> 5f96c356b
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
(cherry picked from commit aaece0ba5e399f248c8255fe509cdb1a862bf299)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b44c902
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b44c902
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b44c902
Branch: refs/heads/branch-1.3
Commit: 4b44c9025fd21b1aab10579c5c4c169003c983a8
Parents: 1a36db1
Author: Esteban Gutierrez
Authored: Fri Aug 11 12:56:20 2017 -0500
Committer: Sean Busbey
Committed: Mon Feb 26 09:47:18 2018 -0600
--
.../hadoop/hbase/master/CatalogJanitor.java | 4 +
.../hadoop/hbase/master/RegionStates.java | 6 +
.../hadoop/hbase/master/ServerManager.java | 7 +
.../TestCatalogJanitorInMemoryStates.java | 188 +++
4 files changed, 205 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 48105c8..893b108 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -216,6 +216,8 @@ public class CatalogJanitor extends ScheduledChore {
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
regionB);
MetaTableAccessor.deleteMergeQualifiers(services.getConnection(),
mergedRegion);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+ services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
services.getServerManager().removeRegion(regionA);
services.getServerManager().removeRegion(regionB);
return true;
@@ -350,6 +352,8 @@ public class CatalogJanitor extends ScheduledChore {
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " +
parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs,
parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
+ if (services.getAssignmentManager().getRegionStates() != null)
+services.getAssignmentManager().getRegionStates().deleteRegion(parent);
services.getServerManager().removeRegion(parent);
result = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 9fda8da..406f5e6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -858,6 +858,12 @@ public class RegionStates {
}
}
+ @VisibleForTesting
+ public boolean isRegionInRegionStates(final HRegionInfo hri) {
+return (getRegionState(hri) != null || isRegionOnline(hri)) ||
isRegionInTransition(hri)
+|| isRegionInState(hri, State.OFFLINE, State.CLOSED);
+ }
+
/**
* Checking if a region was assigned to a server which is not online now.
* If so, we should hold re-assign this region till SSH has split its wals.
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 7ea5fd7..c814c2b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1223,6 +1223,13 @@ public class ServerManager {
flushedSequenceIdByRegion.remove(encodedName);
}
+ @VisibleForTesting
+ public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+final byte[] encodedName = hri.getEncodedNameAsBytes();
+return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+|| flushedSequenceIdByRegion.containsKey(encodedName));
+ }
+
/**
* Cal
[5/6] hbase git commit: HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap.
HBASE-18625 Splitting of region with replica, doesn't update region list in
serverHolding. A server crash leads to overlap.
Signed-off-by: ramkrishna.s.vasudevan
(cherry picked from commit 80d183a707c6d0203da1565f6a7ca3edf4bb2291)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b795553
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b795553
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b795553
Branch: refs/heads/branch-1.2
Commit: 4b795553ea5a05d16088db08c9cc4994e354d56c
Parents: 688d4de
Author: huaxiang sun
Authored: Wed Jan 3 12:24:05 2018 -0800
Committer: Sean Busbey
Committed: Mon Feb 26 10:46:54 2018 -0600
--
.../hadoop/hbase/master/RegionStates.java | 33 ++-
.../TestCatalogJanitorInMemoryStates.java | 91 +---
2 files changed, 112 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b795553/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index cf68a02..3b89f7b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -545,6 +545,19 @@ public class RegionStates {
}
/**
+ * Used in some unit tests
+ */
+ @VisibleForTesting
+ synchronized boolean existsInServerHoldings(final ServerName serverName,
+ final HRegionInfo hri) {
+Set oldRegions = serverHoldings.get(serverName);
+if (oldRegions != null) {
+ return oldRegions.contains(hri);
+}
+return false;
+ }
+
+ /**
* A dead server's wals have been split so that all the regions
* used to be open on it can be safely assigned now. Mark them assignable.
*/
@@ -613,8 +626,26 @@ public class RegionStates {
deleteRegion(hri);
return;
}
+
+/*
+ * One tricky case, if region here is a replica region and its parent is at
+ * SPLIT state, its newState should be same as its parent, not OFFLINE.
+ */
State newState =
- expectedState == null ? State.OFFLINE : expectedState;
+expectedState == null ? State.OFFLINE : expectedState;
+
+if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) {
+ RegionState primateState = getRegionState(
+ RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
+ if ((primateState != null) && (primateState.getState() == State.SPLIT)) {
+if (LOG.isDebugEnabled()) {
+ LOG.debug("Update region " + hri + "to SPLIT, from primary region " +
+ RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
+}
+newState = State.SPLIT;
+ }
+}
+
updateRegionState(hri, newState);
String encodedName = hri.getEncodedName();
synchronized (this) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b795553/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
index d2bed9b..6ea7e2f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
@@ -18,21 +18,20 @@
*/
package org.apache.hadoop.hbase.master;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.master.AssignmentManager;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.AfterClass;
@@ -44,12 +43,6 @@ import org.junit.experimental.categories.Cate
hbase git commit: for creating patch HBASE-20074-V01.patch
Repository: hbase
Updated Branches:
refs/heads/branch-2 372c68fca -> a312705db
for creating patch HBASE-20074-V01.patch
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a312705d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a312705d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a312705d
Branch: refs/heads/branch-2
Commit: a312705dbc8e6d604adcc874526294c72b8ff580
Parents: 372c68f
Author: gsheffi
Authored: Mon Feb 26 11:18:38 2018 +0200
Committer: Michael Stack
Committed: Mon Feb 26 09:56:44 2018 -0800
--
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a312705d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 44b40eb..d60b049 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
-// multiply by a factor (different factors for different index types)
-if (indexType == IndexType.ARRAY_MAP) {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+// multiply by a factor (the same factor for all index types)
+factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-} else {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
- IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-}
+
inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to {} and immutable
segments index to type={}",
StringUtils.byteDesc(inmemoryFlushSize), indexType);
hbase git commit: for creating patch HBASE-20074-V01.patch
Repository: hbase
Updated Branches:
refs/heads/master 7cfb46432 -> 73028d5bd
for creating patch HBASE-20074-V01.patch
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73028d5b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73028d5b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73028d5b
Branch: refs/heads/master
Commit: 73028d5bd9f85655b284654579ddcbbca31e41e8
Parents: 7cfb464
Author: gsheffi
Authored: Mon Feb 26 11:18:38 2018 +0200
Committer: Michael Stack
Committed: Mon Feb 26 09:57:22 2018 -0800
--
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/73028d5b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 44b40eb..d60b049 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
-// multiply by a factor (different factors for different index types)
-if (indexType == IndexType.ARRAY_MAP) {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+// multiply by a factor (the same factor for all index types)
+factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-} else {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
- IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-}
+
inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to {} and immutable
segments index to type={}",
StringUtils.byteDesc(inmemoryFlushSize), indexType);
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server
Repository: hbase
Updated Branches:
refs/heads/branch-2 a312705db -> 44544c7db
HBASE-20069 fix existing findbugs errors in hbase-server
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/44544c7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/44544c7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/44544c7d
Branch: refs/heads/branch-2
Commit: 44544c7db0069683be6e7d8d6adfcdd2a8c40049
Parents: a312705
Author: Michael Stack
Authored: Sat Feb 24 13:01:02 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 10:55:53 2018 -0800
--
.../hbase/io/encoding/EncodedDataBlock.java | 19 +--
.../apache/hadoop/hbase/nio/MultiByteBuff.java | 4 +-
.../hadoop/hbase/nio/TestMultiByteBuff.java | 19 +++
.../hbase/procedure2/ProcedureExecutor.java | 1 -
.../hbase/procedure2/StateMachineProcedure.java | 1 -
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 3 +
.../org/apache/hadoop/hbase/master/HMaster.java | 3 -
.../master/assignment/AssignmentManager.java| 1 -
.../assignment/SplitTableRegionProcedure.java | 7 +-
.../hbase/master/cleaner/CleanerChore.java | 39 ---
.../hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/regionserver/MemStoreFlusher.java | 115 +--
.../hbase/regionserver/RSRpcServices.java | 1 -
.../regionserver/RegionCoprocessorHost.java | 2 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 4 +-
.../hbase/util/compaction/MajorCompactor.java | 9 +-
16 files changed, 147 insertions(+), 84 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/44544c7d/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
index a791c09..af68656 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
@@ -228,6 +228,7 @@ public class EncodedDataBlock {
*/
public byte[] encodeData() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
+byte [] baosBytes = null;
try {
baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
DataOutputStream out = new DataOutputStream(baos);
@@ -255,25 +256,17 @@ public class EncodedDataBlock {
kv.setSequenceId(memstoreTS);
this.dataBlockEncoder.encode(kv, encodingCtx, out);
}
- BufferGrabbingByteArrayOutputStream stream = new
BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- this.dataBlockEncoder.endBlockEncoding(encodingCtx, out,
stream.ourBytes);
+ // Below depends on BAOS internal behavior. toByteArray makes a copy of
bytes so far.
+ baos.flush();
+ baosBytes = baos.toByteArray();
+ this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, baosBytes);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Bug in encoding part of algorithm %s. " +
"Probably it requested more bytes than are available.",
toString()), e);
}
-return baos.toByteArray();
- }
-
- private static class BufferGrabbingByteArrayOutputStream extends
ByteArrayOutputStream {
-private byte[] ourBytes;
-
-@Override
-public synchronized void write(byte[] b, int off, int len) {
- this.ourBytes = b;
-}
+return baosBytes;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/44544c7d/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
index fecf012..847e2eb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
@@ -282,7 +282,7 @@ public class MultiByteBuff extends ByteBuff {
return ByteBufferUtils.toShort(item, offsetInItem);
}
if (items.length - 1 == itemIndex) {
- // means cur item is the last one and we wont be able to read a int.
Throw exception
+ // means cur item is the last one and we wont be able to read a short.
Throw exception
throw new BufferUnderflowException();
}
ByteBuffer nextItem = items[itemIndex + 1];
@@ -294,7 +294,7 @@ public class MultiByteBuff extends ByteBuff {
}
for (int i = 0; i < Bytes.SIZEOF_SHORT - re
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server
Repository: hbase
Updated Branches:
refs/heads/master 73028d5bd -> b11e50666
HBASE-20069 fix existing findbugs errors in hbase-server
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b11e5066
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b11e5066
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b11e5066
Branch: refs/heads/master
Commit: b11e506664614c243c08949c256430d4dd13ba6c
Parents: 73028d5
Author: Michael Stack
Authored: Sat Feb 24 13:01:02 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 16:01:31 2018 -0800
--
.../hbase/io/encoding/EncodedDataBlock.java | 19 +--
.../apache/hadoop/hbase/nio/MultiByteBuff.java | 4 +-
.../hadoop/hbase/nio/TestMultiByteBuff.java | 19 +++
.../hbase/procedure2/ProcedureExecutor.java | 1 -
.../hbase/procedure2/StateMachineProcedure.java | 1 -
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 3 +
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../master/assignment/AssignmentManager.java| 1 -
.../assignment/SplitTableRegionProcedure.java | 7 +-
.../hbase/master/cleaner/CleanerChore.java | 39 ---
.../hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/regionserver/MemStoreFlusher.java | 115 +--
.../hbase/regionserver/RSRpcServices.java | 1 -
.../regionserver/RegionCoprocessorHost.java | 2 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 4 +-
.../hbase/util/compaction/MajorCompactor.java | 9 +-
16 files changed, 147 insertions(+), 82 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
index a791c09..af68656 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
@@ -228,6 +228,7 @@ public class EncodedDataBlock {
*/
public byte[] encodeData() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
+byte [] baosBytes = null;
try {
baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
DataOutputStream out = new DataOutputStream(baos);
@@ -255,25 +256,17 @@ public class EncodedDataBlock {
kv.setSequenceId(memstoreTS);
this.dataBlockEncoder.encode(kv, encodingCtx, out);
}
- BufferGrabbingByteArrayOutputStream stream = new
BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- this.dataBlockEncoder.endBlockEncoding(encodingCtx, out,
stream.ourBytes);
+ // Below depends on BAOS internal behavior. toByteArray makes a copy of
bytes so far.
+ baos.flush();
+ baosBytes = baos.toByteArray();
+ this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, baosBytes);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Bug in encoding part of algorithm %s. " +
"Probably it requested more bytes than are available.",
toString()), e);
}
-return baos.toByteArray();
- }
-
- private static class BufferGrabbingByteArrayOutputStream extends
ByteArrayOutputStream {
-private byte[] ourBytes;
-
-@Override
-public synchronized void write(byte[] b, int off, int len) {
- this.ourBytes = b;
-}
+return baosBytes;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
index fecf012..847e2eb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
@@ -282,7 +282,7 @@ public class MultiByteBuff extends ByteBuff {
return ByteBufferUtils.toShort(item, offsetInItem);
}
if (items.length - 1 == itemIndex) {
- // means cur item is the last one and we wont be able to read a int.
Throw exception
+ // means cur item is the last one and we wont be able to read a short.
Throw exception
throw new BufferUnderflowException();
}
ByteBuffer nextItem = items[itemIndex + 1];
@@ -294,7 +294,7 @@ public class MultiByteBuff extends ByteBuff {
}
for (int i = 0; i < Bytes.SIZEOF_SHORT - remain
hbase git commit: HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region
Repository: hbase
Updated Branches:
refs/heads/branch-1.4 8d3fc596c -> afe892fd4
HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup
region
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/afe892fd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/afe892fd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/afe892fd
Branch: refs/heads/branch-1.4
Commit: afe892fd4d31a2bfe9c8ed903d48adac8e4ce8d5
Parents: 8d3fc59
Author: Thiruvel Thirumoolan
Authored: Mon Feb 26 11:35:09 2018 -0800
Committer: tedyu
Committed: Mon Feb 26 18:01:57 2018 -0800
--
.../hadoop/hbase/master/AssignmentManager.java | 32 +++-
.../hadoop/hbase/master/RegionStates.java | 42 +++--
.../TestRegionMergeTransactionOnCluster.java| 183 ++-
.../TestSplitTransactionOnCluster.java | 134 --
4 files changed, 316 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/afe892fd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index c7339ef..bc1e02f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -255,10 +255,10 @@ public class AssignmentManager extends ZooKeeperListener {
private final RegionStateStore regionStateStore;
/**
- * For testing only! Set to true to skip handling of split.
+ * For testing only! Set to true to skip handling of split and merge.
*/
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL")
- public static boolean TEST_SKIP_SPLIT_HANDLING = false;
+ private static boolean TEST_SKIP_SPLIT_HANDLING = false;
+ private static boolean TEST_SKIP_MERGE_HANDLING = false;
/** Listeners that are called on assignment events. */
private List listeners = new
CopyOnWriteArrayList();
@@ -4054,6 +4054,11 @@ public class AssignmentManager extends ZooKeeperListener
{
mergingRegions.put(encodedName,
new PairOfSameType(a, b));
} else if (code == TransitionCode.MERGED) {
+
+ if (TEST_SKIP_MERGE_HANDLING) {
+return "Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for
merge parent: " + p;
+ }
+
mergingRegions.remove(encodedName);
regionOffline(a, State.MERGED);
regionOffline(b, State.MERGED);
@@ -4180,6 +4185,11 @@ public class AssignmentManager extends ZooKeeperListener
{
regionStates.updateRegionState(hri_b, State.MERGING);
regionStates.updateRegionState(p, State.MERGING_NEW, sn);
+ if (TEST_SKIP_MERGE_HANDLING) {
+LOG.warn("Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for
merge parent: " + p);
+return true; // return true so that the merging node stays
+ }
+
if (et != EventType.RS_ZK_REGION_MERGED) {
this.mergingRegions.put(encodedName,
new PairOfSameType(hri_a, hri_b));
@@ -4727,4 +4737,20 @@ public class AssignmentManager extends ZooKeeperListener
{
threadPoolExecutorService.submit(callable);
}
}
+
+ /*
+ * This is only used for unit-testing split failures.
+ */
+ @VisibleForTesting
+ public static void setTestSkipSplitHandling(boolean skipSplitHandling) {
+TEST_SKIP_SPLIT_HANDLING = skipSplitHandling;
+ }
+
+ /*
+ * This is only used for unit-testing merge failures.
+ */
+ @VisibleForTesting
+ public static void setTestSkipMergeHandling(boolean skipMergeHandling) {
+TEST_SKIP_MERGE_HANDLING = skipMergeHandling;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/afe892fd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 7ce6257..f47d555 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ConfigUtil;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.
hbase git commit: HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region
Repository: hbase
Updated Branches:
refs/heads/branch-1 a37c91bb4 -> ece68d3c6
HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup
region
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ece68d3c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ece68d3c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ece68d3c
Branch: refs/heads/branch-1
Commit: ece68d3c650381e0b7981b685f3e9f2f48b5dec5
Parents: a37c91b
Author: Thiruvel Thirumoolan
Authored: Mon Feb 26 11:35:09 2018 -0800
Committer: tedyu
Committed: Mon Feb 26 18:01:28 2018 -0800
--
.../hadoop/hbase/master/AssignmentManager.java | 32 +++-
.../hadoop/hbase/master/RegionStates.java | 42 +++--
.../TestRegionMergeTransactionOnCluster.java| 183 ++-
.../TestSplitTransactionOnCluster.java | 134 --
4 files changed, 316 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ece68d3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index c7339ef..bc1e02f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -255,10 +255,10 @@ public class AssignmentManager extends ZooKeeperListener {
private final RegionStateStore regionStateStore;
/**
- * For testing only! Set to true to skip handling of split.
+ * For testing only! Set to true to skip handling of split and merge.
*/
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL")
- public static boolean TEST_SKIP_SPLIT_HANDLING = false;
+ private static boolean TEST_SKIP_SPLIT_HANDLING = false;
+ private static boolean TEST_SKIP_MERGE_HANDLING = false;
/** Listeners that are called on assignment events. */
private List listeners = new
CopyOnWriteArrayList();
@@ -4054,6 +4054,11 @@ public class AssignmentManager extends ZooKeeperListener
{
mergingRegions.put(encodedName,
new PairOfSameType(a, b));
} else if (code == TransitionCode.MERGED) {
+
+ if (TEST_SKIP_MERGE_HANDLING) {
+return "Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for
merge parent: " + p;
+ }
+
mergingRegions.remove(encodedName);
regionOffline(a, State.MERGED);
regionOffline(b, State.MERGED);
@@ -4180,6 +4185,11 @@ public class AssignmentManager extends ZooKeeperListener
{
regionStates.updateRegionState(hri_b, State.MERGING);
regionStates.updateRegionState(p, State.MERGING_NEW, sn);
+ if (TEST_SKIP_MERGE_HANDLING) {
+LOG.warn("Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for
merge parent: " + p);
+return true; // return true so that the merging node stays
+ }
+
if (et != EventType.RS_ZK_REGION_MERGED) {
this.mergingRegions.put(encodedName,
new PairOfSameType(hri_a, hri_b));
@@ -4727,4 +4737,20 @@ public class AssignmentManager extends ZooKeeperListener
{
threadPoolExecutorService.submit(callable);
}
}
+
+ /*
+ * This is only used for unit-testing split failures.
+ */
+ @VisibleForTesting
+ public static void setTestSkipSplitHandling(boolean skipSplitHandling) {
+TEST_SKIP_SPLIT_HANDLING = skipSplitHandling;
+ }
+
+ /*
+ * This is only used for unit-testing merge failures.
+ */
+ @VisibleForTesting
+ public static void setTestSkipMergeHandling(boolean skipMergeHandling) {
+TEST_SKIP_MERGE_HANDLING = skipMergeHandling;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ece68d3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 7ce6257..f47d555 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ConfigUtil;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbas
hbase git commit: HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Repository: hbase
Updated Branches:
refs/heads/master b11e50666 -> d3aefe783
HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3aefe78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3aefe78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3aefe78
Branch: refs/heads/master
Commit: d3aefe783476e860e7b1c474b50cf18a7ae0be00
Parents: b11e506
Author: tedyu
Authored: Mon Feb 26 18:29:35 2018 -0800
Committer: tedyu
Committed: Mon Feb 26 18:29:35 2018 -0800
--
.../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3aefe78/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a5fa2c7..5a63ef4 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -527,7 +527,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
Histogram.class, // yammer metrics
- ObjectMapper.class); // jackson-mapper-asl
+ ObjectMapper.class, // jackson-mapper-asl
+ FilterAllFilter.class // hbase-server tests jar
+ );
TableMapReduceUtil.initCredentials(job);
hbase git commit: HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Repository: hbase
Updated Branches:
refs/heads/branch-2 44544c7db -> b3ae58021
HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3ae5802
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3ae5802
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3ae5802
Branch: refs/heads/branch-2
Commit: b3ae58021607e44f99ac3b31dbe716e242a45110
Parents: 44544c7
Author: tedyu
Authored: Mon Feb 26 18:30:09 2018 -0800
Committer: tedyu
Committed: Mon Feb 26 18:30:09 2018 -0800
--
.../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3ae5802/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index b2ebaad..95d363f 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -527,7 +527,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
Histogram.class, // yammer metrics
- ObjectMapper.class); // jackson-mapper-asl
+ ObjectMapper.class, // jackson-mapper-asl
+ FilterAllFilter.class // hbase-server tests jar
+ );
TableMapReduceUtil.initCredentials(job);
hbase git commit: HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing.
Repository: hbase Updated Branches: refs/heads/branch-1.4 afe892fd4 -> c5bc25a7d HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing. Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c5bc25a7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c5bc25a7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c5bc25a7 Branch: refs/heads/branch-1.4 Commit: c5bc25a7d759adbb99f30217f97794046f6c7c24 Parents: afe892f Author: Artem Ervits Authored: Mon Feb 26 18:19:21 2018 -0500 Committer: tedyu Committed: Mon Feb 26 19:29:05 2018 -0800 -- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c5bc25a7/hbase-shaded/hbase-shaded-check-invariants/pom.xml -- diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7f7eb4f..e77fdfb 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -161,6 +161,7 @@ org.codehaus.mojo exec-maven-plugin +1.6.0 check-jar-contents
hbase git commit: HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing.
Repository: hbase Updated Branches: refs/heads/branch-1 ece68d3c6 -> 8e5b8f22f HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing. Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e5b8f22 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e5b8f22 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e5b8f22 Branch: refs/heads/branch-1 Commit: 8e5b8f22fc967705cdc2b3ea724e8713ef3d7eff Parents: ece68d3 Author: Artem Ervits Authored: Mon Feb 26 18:19:21 2018 -0500 Committer: tedyu Committed: Mon Feb 26 19:29:36 2018 -0800 -- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8e5b8f22/hbase-shaded/hbase-shaded-check-invariants/pom.xml -- diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 01587db..67faaa4 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -161,6 +161,7 @@ org.codehaus.mojo exec-maven-plugin +1.6.0 check-jar-contents
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review
Repository: hbase
Updated Branches:
refs/heads/branch-2 b3ae58021 -> 43d661b00
HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address
review
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/43d661b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/43d661b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/43d661b0
Branch: refs/heads/branch-2
Commit: 43d661b008cda46604ddde2187945bb0caa5bc5f
Parents: b3ae580
Author: Michael Stack
Authored: Mon Feb 26 16:26:36 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 23:15:43 2018 -0800
--
.../hadoop/hbase/master/ClusterStatusPublisher.java | 6 +-
.../hadoop/hbase/regionserver/MemStoreFlusher.java| 14 +++---
.../hbase/regionserver/RegionCoprocessorHost.java | 4 +++-
3 files changed, 11 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 5e97204..21fa263 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -134,7 +134,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
@Override
protected void chore() {
-if (!connected) {
+if (!isConnected()) {
return;
}
@@ -170,6 +170,10 @@ public class ClusterStatusPublisher extends ScheduledChore
{
publisher.close();
}
+ private synchronized boolean isConnected() {
+return this.connected;
+ }
+
/**
* Create the dead server to send. A dead server is sent NB_SEND times. We
send at max
* MAX_SERVER_PER_MESSAGE at a time. if there are too many dead servers, we
send the newly
http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index a0e65ec..23321e8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -90,17 +90,9 @@ class MemStoreFlusher implements FlushRequester {
private FlushType flushType;
/**
- * Singleton instance of this class inserted into flush queue.
+ * Singleton instance inserted into flush queue used for signaling.
*/
- private static final WakeupFlushThread WAKEUPFLUSH_INSTANCE = new
WakeupFlushThread();
-
- /**
- * Marker class used as a token inserted into flush queue that ensures the
flusher does not sleep.
- * Create a single instance only.
- */
- private static final class WakeupFlushThread implements FlushQueueEntry {
-private WakeupFlushThread() {}
-
+ private static final FlushQueueEntry WAKEUPFLUSH_INSTANCE = new
FlushQueueEntry() {
@Override
public long getDelay(TimeUnit unit) {
return 0;
@@ -120,7 +112,7 @@ class MemStoreFlusher implements FlushRequester {
public int hashCode() {
return 42;
}
- }
+ };
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index f3c93dc..47b389a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1119,6 +1119,8 @@ public class RegionCoprocessorHost
* @return true or false to return to client if default processing should be
bypassed, or null
* otherwise
*/
+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_BOOLEAN_RETURN_NULL",
+ justification="Null is legit")
public Boolean preCheckAndPutAfterRowLock(
final byte[] row, final byte[] family, final byte[] qualifier, final
CompareOperator op,
final ByteArrayComparable comparator, final Put put) throws IOException {
@@ -1207,7 +12
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review
Repository: hbase
Updated Branches:
refs/heads/master d3aefe783 -> d272ac908
HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address
review
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d272ac90
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d272ac90
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d272ac90
Branch: refs/heads/master
Commit: d272ac908ceb4696e05431066ae02d953fa6fc9d
Parents: d3aefe7
Author: Michael Stack
Authored: Mon Feb 26 16:26:36 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 23:16:21 2018 -0800
--
.../hadoop/hbase/master/ClusterStatusPublisher.java | 6 +-
.../hadoop/hbase/regionserver/MemStoreFlusher.java| 14 +++---
.../hbase/regionserver/RegionCoprocessorHost.java | 4 +++-
3 files changed, 11 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 5e97204..21fa263 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -134,7 +134,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
@Override
protected void chore() {
-if (!connected) {
+if (!isConnected()) {
return;
}
@@ -170,6 +170,10 @@ public class ClusterStatusPublisher extends ScheduledChore
{
publisher.close();
}
+ private synchronized boolean isConnected() {
+return this.connected;
+ }
+
/**
* Create the dead server to send. A dead server is sent NB_SEND times. We
send at max
* MAX_SERVER_PER_MESSAGE at a time. if there are too many dead servers, we
send the newly
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index a0e65ec..23321e8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -90,17 +90,9 @@ class MemStoreFlusher implements FlushRequester {
private FlushType flushType;
/**
- * Singleton instance of this class inserted into flush queue.
+ * Singleton instance inserted into flush queue used for signaling.
*/
- private static final WakeupFlushThread WAKEUPFLUSH_INSTANCE = new
WakeupFlushThread();
-
- /**
- * Marker class used as a token inserted into flush queue that ensures the
flusher does not sleep.
- * Create a single instance only.
- */
- private static final class WakeupFlushThread implements FlushQueueEntry {
-private WakeupFlushThread() {}
-
+ private static final FlushQueueEntry WAKEUPFLUSH_INSTANCE = new
FlushQueueEntry() {
@Override
public long getDelay(TimeUnit unit) {
return 0;
@@ -120,7 +112,7 @@ class MemStoreFlusher implements FlushRequester {
public int hashCode() {
return 42;
}
- }
+ };
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index f3c93dc..47b389a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1119,6 +1119,8 @@ public class RegionCoprocessorHost
* @return true or false to return to client if default processing should be
bypassed, or null
* otherwise
*/
+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_BOOLEAN_RETURN_NULL",
+ justification="Null is legit")
public Boolean preCheckAndPutAfterRowLock(
final byte[] row, final byte[] family, final byte[] qualifier, final
CompareOperator op,
final ByteArrayComparable comparator, final Put put) throws IOException {
@@ -1207,7 +1209,7
hbase git commit: HBASE-20066 Region sequence id may go backward after split or merge
Repository: hbase
Updated Branches:
refs/heads/master d272ac908 -> f06a89b53
HBASE-20066 Region sequence id may go backward after split or merge
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f06a89b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f06a89b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f06a89b5
Branch: refs/heads/master
Commit: f06a89b5317a8e3d4fb220eb462b083bc6e0b5c8
Parents: d272ac9
Author: zhangduo
Authored: Mon Feb 26 20:22:57 2018 +0800
Committer: zhangduo
Committed: Tue Feb 27 15:33:07 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 22 ++-
.../assignment/MergeTableRegionsProcedure.java | 181 +--
.../assignment/SplitTableRegionProcedure.java | 150 +++
.../AbstractStateMachineTableProcedure.java | 14 +-
.../hadoop/hbase/regionserver/HRegion.java | 18 +-
.../apache/hadoop/hbase/wal/WALSplitter.java| 109 ++-
.../TestSequenceIdMonotonicallyIncreasing.java | 156
.../hadoop/hbase/master/AbstractTestDLS.java| 36
.../hbase/wal/TestReadWriteSeqIdFiles.java | 95 ++
9 files changed, 504 insertions(+), 277 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 1ab51e5..e785f96 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -241,11 +241,12 @@ enum SplitTableRegionState {
SPLIT_TABLE_REGION_PRE_OPERATION = 2;
SPLIT_TABLE_REGION_CLOSE_PARENT_REGION = 3;
SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS = 4;
- SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 5;
- SPLIT_TABLE_REGION_UPDATE_META = 6;
- SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 7;
- SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 8;
- SPLIT_TABLE_REGION_POST_OPERATION = 9;
+ SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE = 5;
+ SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 6;
+ SPLIT_TABLE_REGION_UPDATE_META = 7;
+ SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 8;
+ SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 9;
+ SPLIT_TABLE_REGION_POST_OPERATION = 10;
}
message SplitTableRegionStateData {
@@ -260,11 +261,12 @@ enum MergeTableRegionsState {
MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION = 3;
MERGE_TABLE_REGIONS_CLOSE_REGIONS = 4;
MERGE_TABLE_REGIONS_CREATE_MERGED_REGION = 5;
- MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 6;
- MERGE_TABLE_REGIONS_UPDATE_META = 7;
- MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 8;
- MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 9;
- MERGE_TABLE_REGIONS_POST_OPERATION = 10;
+ MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE = 6;
+ MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 7;
+ MERGE_TABLE_REGIONS_UPDATE_META = 8;
+ MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 9;
+ MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 10;
+ MERGE_TABLE_REGIONS_POST_OPERATION = 11;
}
message MergeTableRegionsStateData {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 1c448dc..7c041e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -204,55 +205,55 @@ public class MergeTableRegionsProcedure
}
@Override
- protected Flow executeFromState(
- final MasterProcedureEnv env,
- final MergeTableRegionsState state)
- throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
-if (LOG.isDebugEnabled()) {
- LOG.debug(this + " execute state=" + state);
-}
+ protected Flow executeFromState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
hbase git commit: HBASE-20066 Region sequence id may go backward after split or merge
Repository: hbase
Updated Branches:
refs/heads/branch-2 43d661b00 -> 8e8e50683
HBASE-20066 Region sequence id may go backward after split or merge
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e8e5068
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e8e5068
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e8e5068
Branch: refs/heads/branch-2
Commit: 8e8e50683d23124c4a8dbcb06a338dfa03ca7f85
Parents: 43d661b
Author: zhangduo
Authored: Mon Feb 26 20:22:57 2018 +0800
Committer: zhangduo
Committed: Tue Feb 27 15:37:32 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 22 ++-
.../assignment/MergeTableRegionsProcedure.java | 181 +--
.../assignment/SplitTableRegionProcedure.java | 150 +++
.../AbstractStateMachineTableProcedure.java | 14 +-
.../hadoop/hbase/regionserver/HRegion.java | 18 +-
.../apache/hadoop/hbase/wal/WALSplitter.java| 103 +--
.../TestSequenceIdMonotonicallyIncreasing.java | 156
.../hadoop/hbase/master/AbstractTestDLS.java| 36
.../hbase/wal/TestReadWriteSeqIdFiles.java | 95 ++
9 files changed, 501 insertions(+), 274 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e8e5068/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index f988224..5b42ca9 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -240,11 +240,12 @@ enum SplitTableRegionState {
SPLIT_TABLE_REGION_PRE_OPERATION = 2;
SPLIT_TABLE_REGION_CLOSE_PARENT_REGION = 3;
SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS = 4;
- SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 5;
- SPLIT_TABLE_REGION_UPDATE_META = 6;
- SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 7;
- SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 8;
- SPLIT_TABLE_REGION_POST_OPERATION = 9;
+ SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE = 5;
+ SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 6;
+ SPLIT_TABLE_REGION_UPDATE_META = 7;
+ SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 8;
+ SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 9;
+ SPLIT_TABLE_REGION_POST_OPERATION = 10;
}
message SplitTableRegionStateData {
@@ -259,11 +260,12 @@ enum MergeTableRegionsState {
MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION = 3;
MERGE_TABLE_REGIONS_CLOSE_REGIONS = 4;
MERGE_TABLE_REGIONS_CREATE_MERGED_REGION = 5;
- MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 6;
- MERGE_TABLE_REGIONS_UPDATE_META = 7;
- MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 8;
- MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 9;
- MERGE_TABLE_REGIONS_POST_OPERATION = 10;
+ MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE = 6;
+ MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 7;
+ MERGE_TABLE_REGIONS_UPDATE_META = 8;
+ MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 9;
+ MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 10;
+ MERGE_TABLE_REGIONS_POST_OPERATION = 11;
}
message MergeTableRegionsStateData {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e8e5068/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 1c448dc..7c041e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -204,55 +205,55 @@ public class MergeTableRegionsProcedure
}
@Override
- protected Flow executeFromState(
- final MasterProcedureEnv env,
- final MergeTableRegionsState state)
- throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
-if (LOG.isDebugEnabled()) {
- LOG.debug(this + " execute state=" + state);
-}
+ protected Flow executeFromState(final MasterProcedureEnv env, final
MergeTableRegionsState sta
