hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
Repository: hbase Updated Branches: refs/heads/master a34f129af -> 8c74d177f HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c74d177 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c74d177 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c74d177 Branch: refs/heads/master Commit: 8c74d177f68bbd5412cef96dc33f16ba33ff7875 Parents: a34f129 Author: zhangduo Authored: Mon Feb 26 10:22:09 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:50:51 2018 +0800 -- .../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++ .../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java index 1253ef9..ba9bcdc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java @@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends Procedure private PeerOperationType type; + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "IS2_INCONSISTENT_SYNC", + justification = "Will never change after construction") private ServerName targetServer; private boolean dispatched; http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index 4e1b20d..ebb8042 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -403,7 +403,7 @@ public class ReplicationSourceManager implements ReplicationListener { toStartup.add(replicationSource); } } -for (ReplicationSourceInterface replicationSource : oldsources) { +for (ReplicationSourceInterface replicationSource : toStartup) { replicationSource.startup(); } }
[08/50] [abbrv] hbase git commit: HBASE-20043 ITBLL fails against hadoop3
HBASE-20043 ITBLL fails against hadoop3 Fix MoveRandomRegionOfTableAction. It depended on old AM behavior. Make it do explicit move as is required in AMv3; w/o it, it was just closing region causing test to fail. Fix pom so hadoop3 profile specifies a different netty3 version. Bunch of logging format change that came of trying trying to read the spew from this test. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b3ae58e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b3ae58e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b3ae58e Branch: refs/heads/HBASE-19397-branch-2 Commit: 8b3ae58e187f3d731a779528f565c53e40dd7695 Parents: 2d5065b Author: Michael Stack Authored: Fri Feb 23 14:53:52 2018 -0800 Committer: Michael Stack Committed: Sat Feb 24 17:29:24 2018 -0800 -- .../apache/hadoop/hbase/io/ByteBufferPool.java | 5 +-- .../hadoop/hbase/chaos/actions/Action.java | 32 +- .../actions/MoveRandomRegionOfTableAction.java | 13 .../chaos/actions/MoveRegionsOfTableAction.java | 35 .../chaos/monkies/PolicyBasedChaosMonkey.java | 2 +- .../hbase/chaos/policies/PeriodicPolicy.java| 6 ++-- .../policies/PeriodicRandomActionPolicy.java| 3 +- .../procedure2/RemoteProcedureDispatcher.java | 4 +-- .../hadoop/hbase/io/hfile/CacheConfig.java | 2 +- .../hadoop/hbase/io/hfile/LruBlockCache.java| 14 .../apache/hadoop/hbase/ipc/NettyRpcServer.java | 4 +-- .../apache/hadoop/hbase/ipc/RpcExecutor.java| 10 +++--- .../org/apache/hadoop/hbase/master/HMaster.java | 1 - .../RegionServerProcedureManagerHost.java | 8 ++--- .../hbase/regionserver/CompactingMemStore.java | 5 +-- .../hbase/regionserver/CompactionPipeline.java | 2 +- .../hadoop/hbase/regionserver/HRegion.java | 3 +- .../hbase/regionserver/HRegionServer.java | 4 +-- .../hadoop/hbase/regionserver/HStore.java | 2 +- .../hbase/regionserver/HeapMemoryManager.java | 10 +++--- .../hadoop/hbase/regionserver/Leases.java | 3 +- .../MemStoreCompactionStrategy.java | 5 ++- .../hbase/regionserver/MemStoreCompactor.java | 3 +- .../hadoop/hbase/regionserver/Segment.java | 2 +- .../compactions/CompactionConfiguration.java| 9 ++--- .../compactions/ExploringCompactionPolicy.java | 6 ++-- .../PressureAwareThroughputController.java | 2 +- .../replication/regionserver/Replication.java | 2 +- pom.xml | 9 +++-- 29 files changed, 108 insertions(+), 98 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8b3ae58e/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java index 6c009b3..caca20b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java @@ -80,8 +80,9 @@ public class ByteBufferPool { this.maxPoolSize = maxPoolSize; this.directByteBuffer = directByteBuffer; // TODO can add initialPoolSize config also and make those many BBs ready for use. -LOG.info("Created ByteBufferPool with bufferSize : " + bufferSize + " and maxPoolSize : " -+ maxPoolSize); +LOG.info("Created with bufferSize={} and maxPoolSize={}", +org.apache.hadoop.util.StringUtils.byteDesc(bufferSize), +org.apache.hadoop.util.StringUtils.byteDesc(maxPoolSize)); this.count = new AtomicInteger(0); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8b3ae58e/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java -- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java index ae8cd1f..2b2c1b8 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java @@ -138,63 +138,63 @@ public class Action { } protected void killMaster(ServerName server) throws IOException { -LOG.info("Killing master:" + server); +LOG.info("Killing master " + server); cluster.killMaster(server); cluster.waitForMasterToStop(server, killMasterTimeout); -LOG.info("Killed master server:" + server); +LOG.info("Killed master " + server); } protected void startMaster(ServerName server) throws IOException { -LOG.info("Starting master:" + s
[40/50] [abbrv] hbase git commit: HBASE-19719 Fix checkstyle issues
HBASE-19719 Fix checkstyle issues Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8b58c36 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8b58c36 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8b58c36 Branch: refs/heads/HBASE-19397-branch-2 Commit: f8b58c36df94ee195458cb55f4f5d58d8084 Parents: 03984c6 Author: zhangduo Authored: Sat Jan 6 08:30:55 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../hbase/replication/ReplicationStorageFactory.java | 2 +- .../master/assignment/RegionTransitionProcedure.java | 4 ++-- .../hbase/master/procedure/RSProcedureDispatcher.java | 13 ++--- .../master/ReplicationPeerConfigUpgrader.java | 8 4 files changed, 13 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java index 60d0749..462cfed 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java @@ -27,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience; * For now we only have zk based implementation. */ @InterfaceAudience.Private -public class ReplicationStorageFactory { +public final class ReplicationStorageFactory { private ReplicationStorageFactory() { } http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java index 1724a38..8277dbe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java @@ -36,11 +36,11 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; - /** * Base class for the Assign and Unassign Procedure. * http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b58c36/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java index 57a4535..6c78914 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.master.procedure; import java.io.IOException; @@ -36,6 +35,12 @@ import org.apache.hadoop.ipc.RemoteException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap; +import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; +import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; @@ -47,12 +52,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR import org.apache.hadoop.hbase.shaded.protobu
[06/50] [abbrv] hbase git commit: HBASE-19583 update RM list to remove EOM versions
HBASE-19583 update RM list to remove EOM versions Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/675f76b5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/675f76b5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/675f76b5 Branch: refs/heads/HBASE-19397-branch-2 Commit: 675f76b5a4ea5aca9ba659d4426ebfa6f6a21855 Parents: 62b8da9 Author: Sean Busbey Authored: Sat Feb 24 13:03:22 2018 -0600 Committer: Sean Busbey Committed: Sat Feb 24 16:50:30 2018 -0600 -- src/main/asciidoc/_chapters/developer.adoc | 3 --- 1 file changed, 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/675f76b5/src/main/asciidoc/_chapters/developer.adoc -- diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc index ef9ae25..6d959c2 100644 --- a/src/main/asciidoc/_chapters/developer.adoc +++ b/src/main/asciidoc/_chapters/developer.adoc @@ -1539,9 +1539,6 @@ NOTE: End-of-life releases are not included in this list. | Release | Release Manager -| 1.1 -| Nick Dimiduk - | 1.2 | Sean Busbey
[37/50] [abbrv] hbase git commit: HBASE-19697 Remove TestReplicationAdminUsingProcedure
HBASE-19697 Remove TestReplicationAdminUsingProcedure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d448c013 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d448c013 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d448c013 Branch: refs/heads/HBASE-19397-branch-2 Commit: d448c01303e4627ece76c13ab91688fe334bd6ab Parents: 85e85be Author: zhangduo Authored: Wed Jan 3 21:13:57 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:03 2018 +0800 -- .../TestReplicationAdminUsingProcedure.java | 225 --- 1 file changed, 225 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d448c013/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java deleted file mode 100644 index 1300376..000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client.replication; - -import java.io.IOException; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.TestReplicationBase; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; - -@Category({ MediumTests.class, ClientTests.class }) -public class TestReplicationAdminUsingProcedure extends TestReplicationBase { - - private static final String PEER_ID = "2"; - private static final Logger LOG = Logger.getLogger(TestReplicationAdminUsingProcedure.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { -conf1.setInt("hbase.multihconnection.threads.max", 10); - -// Start the master & slave mini cluster. -TestReplicationBase.setUpBeforeClass(); - -// Remove the replication peer -hbaseAdmin.removeReplicationPeer(PEER_ID); - } - - private void loadData(int startRowKey, int endRowKey) throws IOException { -for (int i = startRowKey; i < endRowKey; i++) { - byte[] rowKey = Bytes.add(row, Bytes.toBytes(i)); - Put put = new Put(rowKey); - put.addColumn(famName, null, Bytes.toBytes(i)); - htable1.put(put); -} - } - - private void waitForReplication(int expectedRows, int retries) - throws IOException, InterruptedException { -Scan scan; -for (int i = 0; i < retries; i++) { - scan = new Scan(); - if (i == retries - 1) { -throw new IOException("Waited too much time for normal batch replication"); - } - try (ResultScanner scanner = htable2.getScanner(scan)) { -int count = 0; -for (Result res : scanner) { - count++; -} -if (count != expectedRows) { - LOG.info("Only got " + count + " rows, expected rows: " + expectedRows); - Thread.sleep(SLEEP_TIME); -} else { - return; -} - } -} - } - - @Before - public void setUp() throws IOException { -ReplicationPeerConfig rpc = new ReplicationPeerConfig(); -
[20/50] [abbrv] hbase git commit: HBASE-19630 Add peer cluster key check when add new replication peer
HBASE-19630 Add peer cluster key check when add new replication peer Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3be7732 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3be7732 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3be7732 Branch: refs/heads/HBASE-19397-branch-2 Commit: b3be773239a141928066bbaabda4571a8689d146 Parents: 751a6eb Author: Guanghao Zhang Authored: Tue Dec 26 21:10:00 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../replication/ReplicationPeerManager.java | 54 .../replication/TestReplicationAdmin.java | 22 2 files changed, 54 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b3be7732/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java index 84abfeb..b78cbce 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.replication; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; import org.apache.hadoop.hbase.replication.ReplicationQueueInfo; import org.apache.hadoop.hbase.replication.ReplicationQueueStorage; import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; +import org.apache.hadoop.hbase.zookeeper.ZKConfig; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.yetus.audience.InterfaceAudience; @@ -216,36 +218,36 @@ public final class ReplicationPeerManager { return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty(); } - /** - * If replicate_all flag is true, it means all user tables will be replicated to peer cluster. - * Then allow config exclude namespaces or exclude table-cfs which can't be replicated to peer - * cluster. - * - * If replicate_all flag is false, it means all user tables can't be replicated to peer cluster. - * Then allow to config namespaces or table-cfs which will be replicated to peer cluster. - */ - private static void checkPeerConfig(ReplicationPeerConfig peerConfig) - throws DoNotRetryIOException { + private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws DoNotRetryIOException { +checkClusterKey(peerConfig.getClusterKey()); + if (peerConfig.replicateAllUserTables()) { - if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) || -(peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) { -throw new DoNotRetryIOException("Need clean namespaces or table-cfs config firstly " + - "when you want replicate all cluster"); + // If replicate_all flag is true, it means all user tables will be replicated to peer cluster. + // Then allow config exclude namespaces or exclude table-cfs which can't be replicated to peer + // cluster. + if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) + || (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) { +throw new DoNotRetryIOException("Need clean namespaces or table-cfs config firstly " ++ "when you want replicate all cluster"); } checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(), peerConfig.getExcludeTableCFsMap()); } else { - if ((peerConfig.getExcludeNamespaces() != null && -!peerConfig.getExcludeNamespaces().isEmpty()) || -(peerConfig.getExcludeTableCFsMap() != null && - !peerConfig.getExcludeTableCFsMap().isEmpty())) { + // If replicate_all flag is false, it means all user tables can't be replicated to peer + // cluster. Then allow to config namespaces or table-cfs which will be replicated to peer + // cluster. + if ((peerConfig.getExcludeNamespaces() != null + && !peerConfig.getExcludeNamespaces().isEmpty()) + || (peerConfig.getExcludeTableCFsMap() != null + && !peerConfig.getExcludeTableCFsMap().isEmpty())) { throw new DoNotRetryIOException( -"Need clean exclude-namespaces or exclude-table-cfs config firstly"
[46/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java index 4c1b94f..6d2b578 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java @@ -28,12 +28,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; @@ -41,6 +40,8 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; @@ -71,9 +72,6 @@ public class TestNamespaceReplication extends TestReplicationBase { private static final byte[] val = Bytes.toBytes("myval"); - private static HTableDescriptor tabA; - private static HTableDescriptor tabB; - private static Connection connection1; private static Connection connection2; private static Admin admin1; @@ -93,23 +91,21 @@ public class TestNamespaceReplication extends TestReplicationBase { admin2.createNamespace(NamespaceDescriptor.create(ns1).build()); admin2.createNamespace(NamespaceDescriptor.create(ns2).build()); -tabA = new HTableDescriptor(tabAName); -HColumnDescriptor fam = new HColumnDescriptor(f1Name); -fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); -tabA.addFamily(fam); -fam = new HColumnDescriptor(f2Name); -fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); -tabA.addFamily(fam); +TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tabAName); +builder.addColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()); +builder.addColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()); +TableDescriptor tabA = builder.build(); admin1.createTable(tabA); admin2.createTable(tabA); -tabB = new HTableDescriptor(tabBName); -fam = new HColumnDescriptor(f1Name); -fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); -tabB.addFamily(fam); -fam = new HColumnDescriptor(f2Name); -fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); -tabB.addFamily(fam); +builder = TableDescriptorBuilder.newBuilder(tabBName); +builder.addColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()); +builder.addColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()); +TableDescriptor tabB = builder.build(); admin1.createTable(tabB); admin2.createTable(tabB); } @@ -137,22 +133,24 @@ public class TestNamespaceReplication extends TestReplicationBase { @Test public void testNamespaceReplication() throws Exception { +String peerId = "2"; + Table htab1A = connection1.getTable(tabAName); Table htab2A = connection2.getTable(tabAName); Table htab1B = connection1.getTable(tabBName); Table htab2B = connection2.getTable(tabBName); -ReplicationPeerConfig rpc = admin.getPeerConfig("2"); -rpc.setReplicateAllUserTables(false); -admin.updatePeerConfig("2", rpc); +ReplicationPeerConfig rpc = admin1.getReplicationPeerConfig(peerId); +admin1.updateReplicationPeerConfig(peerId, + ReplicationPeerConfig.newBuilder(rpc).setReplicateAllUserTables(false).build()); // add ns1 to peer config which replicate to cluster2 -rpc = admin.getPeerConfig("2"); +rpc = admin1.getReplicationPeerConfig(peerId); Set namespaces = new HashSet<>(); namespaces.add(ns1); -rpc.setNamespaces(namespaces); -admin.updatePeerConfig("2", rpc); +admin1.updateReplicationPeerConfig(peerId, + Replication
[15/50] [abbrv] hbase git commit: HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure
HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8eb68c40 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8eb68c40 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8eb68c40 Branch: refs/heads/HBASE-19397-branch-2 Commit: 8eb68c409531641ccf21215b1c7a11fa0b3f8bf7 Parents: 4a497cc Author: huzheng Authored: Wed Dec 20 10:47:18 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hadoop/hbase/protobuf/ProtobufUtil.java | 11 +- .../hbase/shaded/protobuf/ProtobufUtil.java | 13 +- .../hbase/replication/ReplicationListener.java | 14 -- .../hbase/replication/ReplicationPeer.java | 28 ++- .../replication/ReplicationPeerZKImpl.java | 186 --- .../replication/ReplicationPeersZKImpl.java | 19 +- .../replication/ReplicationTrackerZKImpl.java | 73 +- .../regionserver/ReplicationSourceService.java | 6 + .../handler/RSProcedureHandler.java | 3 + .../replication/BaseReplicationEndpoint.java| 2 +- .../regionserver/PeerProcedureHandler.java | 38 .../regionserver/PeerProcedureHandlerImpl.java | 81 +++ .../regionserver/RefreshPeerCallable.java | 39 +++- .../replication/regionserver/Replication.java | 9 + .../regionserver/ReplicationSource.java | 8 +- .../regionserver/ReplicationSourceManager.java | 37 ++- .../TestReplicationAdminUsingProcedure.java | 226 +++ .../replication/DummyModifyPeerProcedure.java | 48 .../TestDummyModifyPeerProcedure.java | 80 --- .../TestReplicationTrackerZKImpl.java | 61 - .../TestReplicationSourceManager.java | 32 ++- 21 files changed, 532 insertions(+), 482 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8eb68c40/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 3033da7..f11fcf6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.protobuf; +import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC; + import com.google.protobuf.ByteString; import com.google.protobuf.CodedInputStream; import com.google.protobuf.InvalidProtocolBufferException; @@ -191,7 +193,7 @@ public final class ProtobufUtil { * byte array that is bytes.length plus {@link ProtobufMagic#PB_MAGIC}.length. */ public static byte [] prependPBMagic(final byte [] bytes) { -return Bytes.add(ProtobufMagic.PB_MAGIC, bytes); +return Bytes.add(PB_MAGIC, bytes); } /** @@ -216,10 +218,11 @@ public final class ProtobufUtil { * @param bytes bytes to check * @throws DeserializationException if we are missing the pb magic prefix */ - public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException { + public static void expectPBMagicPrefix(final byte[] bytes) throws DeserializationException { if (!isPBMagicPrefix(bytes)) { - throw new DeserializationException("Missing pb magic " + - Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix"); + String bytesPrefix = bytes == null ? "null" : Bytes.toStringBinary(bytes, 0, PB_MAGIC.length); + throw new DeserializationException( + "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix, bytes: " + bytesPrefix); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/8eb68c40/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 4472846..520a4cd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.shaded.protobuf; +import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -277,7 +279,7 @@ public final class ProtobufUtil { * byte array that is bytes.length plus {@link ProtobufMag
[32/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6bbc6071 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6bbc6071 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6bbc6071 Branch: refs/heads/HBASE-19397-branch-2 Commit: 6bbc60719719579833ffd2a2905009f43a27bdd5 Parents: 296d586 Author: zhangduo Authored: Wed Dec 27 22:03:51 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:24 2018 +0800 -- .../hbase/replication/ReplicationFactory.java | 9 +- .../hbase/replication/ReplicationQueues.java| 161 --- .../replication/ReplicationQueuesArguments.java | 70 --- .../replication/ReplicationQueuesZKImpl.java| 417 - .../hbase/replication/ReplicationTableBase.java | 442 --- .../replication/ReplicationTrackerZKImpl.java | 21 +- .../replication/ZKReplicationQueueStorage.java | 22 + .../replication/TestReplicationStateBasic.java | 131 +++--- .../replication/TestReplicationStateZKImpl.java | 41 +- .../regionserver/DumpReplicationQueues.java | 15 +- .../RecoveredReplicationSource.java | 17 +- .../RecoveredReplicationSourceShipper.java | 22 +- .../replication/regionserver/Replication.java | 20 +- .../regionserver/ReplicationSource.java | 16 +- .../ReplicationSourceInterface.java | 11 +- .../regionserver/ReplicationSourceManager.java | 265 ++- .../regionserver/ReplicationSyncUp.java | 32 +- .../hbase/master/cleaner/TestLogsCleaner.java | 12 +- .../cleaner/TestReplicationHFileCleaner.java| 23 +- .../cleaner/TestReplicationZKNodeCleaner.java | 22 +- .../replication/ReplicationSourceDummy.java | 6 +- .../replication/TestReplicationSyncUpTool.java | 4 +- .../TestReplicationSourceManager.java | 97 ++-- .../TestReplicationSourceManagerZkImpl.java | 57 +-- 24 files changed, 365 insertions(+), 1568 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java index 6c1c213..5e70e57 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java @@ -17,12 +17,11 @@ */ package org.apache.hadoop.hbase.replication; -import org.apache.commons.lang3.reflect.ConstructorUtils; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.yetus.audience.InterfaceAudience; /** * A factory class for instantiating replication objects that deal with replication state. @@ -30,12 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @InterfaceAudience.Private public class ReplicationFactory { - public static ReplicationQueues getReplicationQueues(ReplicationQueuesArguments args) - throws Exception { -return (ReplicationQueues) ConstructorUtils.invokeConstructor(ReplicationQueuesZKImpl.class, - args); - } - public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf, Abortable abortable) { return getReplicationPeers(zk, conf, null, abortable); http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java deleted file mode 100644 index a2d21f7..000 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apa
[12/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
HBASE-19543 Abstract a replication storage interface to extract the zk specific code Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d8f09f1a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d8f09f1a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d8f09f1a Branch: refs/heads/HBASE-19397-branch-2 Commit: d8f09f1aad7dda4cb3ffd1776edcb98c9de54d20 Parents: 8eb68c4 Author: zhangduo Authored: Fri Dec 22 14:37:28 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hadoop/hbase/util/CollectionUtils.java | 3 + hbase-replication/pom.xml | 12 + .../replication/ReplicationPeerStorage.java | 74 .../replication/ReplicationQueueStorage.java| 164 +++ .../replication/ReplicationStateZKBase.java | 1 - .../replication/ReplicationStorageFactory.java | 49 +++ .../replication/ZKReplicationPeerStorage.java | 164 +++ .../replication/ZKReplicationQueueStorage.java | 425 +++ .../replication/ZKReplicationStorageBase.java | 75 .../TestZKReplicationPeerStorage.java | 171 .../TestZKReplicationQueueStorage.java | 171 .../org/apache/hadoop/hbase/master/HMaster.java | 34 +- .../hadoop/hbase/master/MasterServices.java | 6 +- .../master/procedure/MasterProcedureEnv.java| 24 +- .../master/replication/AddPeerProcedure.java| 6 +- .../replication/DisablePeerProcedure.java | 7 +- .../master/replication/EnablePeerProcedure.java | 6 +- .../master/replication/ModifyPeerProcedure.java | 41 +- .../master/replication/RemovePeerProcedure.java | 6 +- .../master/replication/ReplicationManager.java | 199 - .../replication/ReplicationPeerManager.java | 331 +++ .../replication/UpdatePeerConfigProcedure.java | 7 +- .../replication/TestReplicationAdmin.java | 64 ++- .../hbase/master/MockNoopMasterServices.java| 13 +- .../hbase/master/TestMasterNoCluster.java | 3 +- .../TestReplicationDisableInactivePeer.java | 6 +- 26 files changed, 1749 insertions(+), 313 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java index 875b124..8bbb6f1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java @@ -107,6 +107,9 @@ public class CollectionUtils { return list.get(list.size() - 1); } + public static List nullToEmpty(List list) { +return list != null ? list : Collections.emptyList(); + } /** * In HBASE-16648 we found that ConcurrentHashMap.get is much faster than computeIfAbsent if the * value already exists. Notice that the implementation does not guarantee that the supplier will http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-replication/pom.xml -- diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index edce309..77517cc 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -104,6 +104,18 @@ org.apache.hbase hbase-zookeeper + + org.apache.hbase + hbase-common + test-jar + test + + + org.apache.hbase + hbase-zookeeper + test-jar + test + org.apache.commons http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java new file mode 100644 index 000..e00cd0d --- /dev/null +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + *
[28/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index 5b7bab8..ce9882a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -160,7 +160,6 @@ public class ReplicationSourceManager implements ReplicationListener { this.clusterId = clusterId; this.walFileLengthProvider = walFileLengthProvider; this.replicationTracker.registerListener(this); -this.replicationPeers.getAllPeerIds(); // It's preferable to failover 1 RS at a time, but with good zk servers // more could be processed at the same time. int nbWorkers = conf.getInt("replication.executor.workers", 1); @@ -260,8 +259,8 @@ public class ReplicationSourceManager implements ReplicationListener { } List otherRegionServers = replicationTracker.getListOfRegionServers().stream() .map(ServerName::valueOf).collect(Collectors.toList()); -LOG.info( - "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers); +LOG.info("Current list of replicators: " + currentReplicators + " other RSs: " ++ otherRegionServers); // Look if there's anything to process after a restart for (ServerName rs : currentReplicators) { @@ -278,7 +277,7 @@ public class ReplicationSourceManager implements ReplicationListener { * The returned future is for adoptAbandonedQueues task. */ Future init() throws IOException, ReplicationException { -for (String id : this.replicationPeers.getConnectedPeerIds()) { +for (String id : this.replicationPeers.getAllPeerIds()) { addSource(id); if (replicationForBulkLoadDataEnabled) { // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case @@ -297,8 +296,8 @@ public class ReplicationSourceManager implements ReplicationListener { */ @VisibleForTesting ReplicationSourceInterface addSource(String id) throws IOException, ReplicationException { -ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(id); -ReplicationPeer peer = replicationPeers.getConnectedPeer(id); +ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id); +ReplicationPeer peer = replicationPeers.getPeer(id); ReplicationSourceInterface src = getReplicationSource(id, peerConfig, peer); synchronized (this.walsById) { this.sources.add(src); @@ -344,7 +343,7 @@ public class ReplicationSourceManager implements ReplicationListener { public void deleteSource(String peerId, boolean closeConnection) { abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), peerId)); if (closeConnection) { - this.replicationPeers.peerDisconnected(peerId); + this.replicationPeers.removePeer(peerId); } } @@ -437,12 +436,12 @@ public class ReplicationSourceManager implements ReplicationListener { // update replication queues on ZK // synchronize on replicationPeers to avoid adding source for the to-be-removed peer synchronized (replicationPeers) { - for (String id : replicationPeers.getConnectedPeerIds()) { + for (String id : replicationPeers.getAllPeerIds()) { try { this.queueStorage.addWAL(server.getServerName(), id, logName); } catch (ReplicationException e) { - throw new IOException("Cannot add log to replication queue" + -" when creating a new source, queueId=" + id + ", filename=" + logName, e); + throw new IOException("Cannot add log to replication queue" + + " when creating a new source, queueId=" + id + ", filename=" + logName, e); } } } @@ -587,7 +586,7 @@ public class ReplicationSourceManager implements ReplicationListener { public void addPeer(String id) throws ReplicationException, IOException { LOG.info("Trying to add peer, peerId: " + id); -boolean added = this.replicationPeers.peerConnected(id); +boolean added = this.replicationPeers.addPeer(id); if (added) { LOG.info("Peer " + id + " connected success, trying to start the replication source thread."); addSource(id); @@ -723,16 +722,21 @@ public class ReplicationSourceManager implements ReplicationListener { // there is not an actual peer defined corresponding to peerId for the failover. ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
[50/50] [abbrv] hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/732d2279 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/732d2279 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/732d2279 Branch: refs/heads/HBASE-19397-branch-2 Commit: 732d22793808627c1443b054bbc65ba923b37bc3 Parents: 2d5c4be Author: zhangduo Authored: Mon Feb 26 10:22:09 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:53 2018 +0800 -- .../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++ .../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/732d2279/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java index 1253ef9..ba9bcdc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java @@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends Procedure private PeerOperationType type; + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "IS2_INCONSISTENT_SYNC", + justification = "Will never change after construction") private ServerName targetServer; private boolean dispatched; http://git-wip-us.apache.org/repos/asf/hbase/blob/732d2279/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index 6965f55..d11dc8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -402,7 +402,7 @@ public class ReplicationSourceManager implements ReplicationListener { toStartup.add(replicationSource); } } -for (ReplicationSourceInterface replicationSource : oldsources) { +for (ReplicationSourceInterface replicationSource : toStartup) { replicationSource.startup(); } }
[13/50] [abbrv] hbase git commit: HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes
HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4a497cce Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4a497cce Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4a497cce Branch: refs/heads/HBASE-19397-branch-2 Commit: 4a497cced4ad485346e7fd49b7952e05d8be6c1f Parents: de68d00 Author: zhangduo Authored: Thu Dec 21 21:59:46 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hadoop/hbase/master/replication/AddPeerProcedure.java | 6 +++--- .../hadoop/hbase/master/replication/DisablePeerProcedure.java | 6 +++--- .../hadoop/hbase/master/replication/EnablePeerProcedure.java | 6 +++--- .../hadoop/hbase/master/replication/ModifyPeerProcedure.java | 6 +++--- .../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 6 +++--- .../hadoop/hbase/master/replication/RemovePeerProcedure.java | 6 +++--- .../hbase/master/replication/UpdatePeerConfigProcedure.java| 6 +++--- 7 files changed, 21 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java index c3862d8..066c3e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; @@ -28,6 +26,8 @@ import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData; @@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.A @InterfaceAudience.Private public class AddPeerProcedure extends ModifyPeerProcedure { - private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class); + private static final Logger LOG = LoggerFactory.getLogger(AddPeerProcedure.class); private ReplicationPeerConfig peerConfig; http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java index 0b32db9..9a28de6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java @@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The procedure for disabling a replication peer. @@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public class DisablePeerProcedure extends ModifyPeerProcedure { - private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class); + private static final Logger LOG = LoggerFactory.getLogger(DisablePeerProcedure.class); public DisablePeerProcedure() { } http://git-wip-us.apache.org/repos/asf/hbase/blob/4a497cce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java -- diff --git a/hbase-server/src/mai
[48/50] [abbrv] hbase git commit: HBASE-19923 Reset peer state and config when refresh replication source failed
HBASE-19923 Reset peer state and config when refresh replication source failed Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d5c4be1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d5c4be1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d5c4be1 Branch: refs/heads/HBASE-19397-branch-2 Commit: 2d5c4be16fd214f3c118255daacdcd4f64e4476c Parents: 1c4938d Author: Guanghao Zhang Authored: Tue Feb 6 14:58:39 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../hbase/replication/ReplicationPeerImpl.java | 4 ++-- .../regionserver/PeerProcedureHandlerImpl.java | 24 2 files changed, 22 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5c4be1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java index 604e0bb..d656466 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java @@ -54,11 +54,11 @@ public class ReplicationPeerImpl implements ReplicationPeer { this.peerConfigListeners = new ArrayList<>(); } - void setPeerState(boolean enabled) { + public void setPeerState(boolean enabled) { this.peerState = enabled ? PeerState.ENABLED : PeerState.DISABLED; } - void setPeerConfig(ReplicationPeerConfig peerConfig) { + public void setPeerConfig(ReplicationPeerConfig peerConfig) { this.peerConfig = peerConfig; peerConfigListeners.forEach(listener -> listener.peerConfigUpdated(peerConfig)); } http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5c4be1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java index ce8fdae..a02d181 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java @@ -62,18 +62,26 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler { private void refreshPeerState(String peerId) throws ReplicationException, IOException { PeerState newState; Lock peerLock = peersLock.acquireLock(peerId); +ReplicationPeerImpl peer = null; +PeerState oldState = null; +boolean success = false; try { - ReplicationPeerImpl peer = replicationSourceManager.getReplicationPeers().getPeer(peerId); + peer = replicationSourceManager.getReplicationPeers().getPeer(peerId); if (peer == null) { throw new ReplicationException("Peer with id=" + peerId + " is not cached."); } - PeerState oldState = peer.getPeerState(); + oldState = peer.getPeerState(); newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId); // RS need to start work with the new replication state change if (oldState.equals(PeerState.ENABLED) && newState.equals(PeerState.DISABLED)) { replicationSourceManager.refreshSources(peerId); } + success = true; } finally { + if (!success && peer != null) { +// Reset peer state if refresh source failed +peer.setPeerState(oldState.equals(PeerState.ENABLED)); + } peerLock.unlock(); } } @@ -91,19 +99,27 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler { @Override public void updatePeerConfig(String peerId) throws ReplicationException, IOException { Lock peerLock = peersLock.acquireLock(peerId); +ReplicationPeerImpl peer = null; +ReplicationPeerConfig oldConfig = null; +boolean success = false; try { - ReplicationPeerImpl peer = replicationSourceManager.getReplicationPeers().getPeer(peerId); + peer = replicationSourceManager.getReplicationPeers().getPeer(peerId); if (peer == null) { throw new ReplicationException("Peer with id=" + peerId + " is not cached."); } - ReplicationPeerConfig oldConfig = peer.getPeerConfig(); + oldConfig = peer.getPeerConfig(); ReplicationPeerConfig newConfig = repl
[41/50] [abbrv] hbase git commit: HBASE-19936 Introduce a new base class for replication peer procedure
HBASE-19936 Introduce a new base class for replication peer procedure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1c4938da Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1c4938da Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1c4938da Branch: refs/heads/HBASE-19397-branch-2 Commit: 1c4938dab8730eb596ecfa0ab414334887fd14a4 Parents: 0502464 Author: zhangduo Authored: Mon Feb 5 16:14:25 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../src/main/protobuf/MasterProcedure.proto | 2 +- .../replication/AbstractPeerProcedure.java | 97 .../master/replication/ModifyPeerProcedure.java | 67 +- 3 files changed, 102 insertions(+), 64 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1c4938da/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index e45e12f..726217f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -399,7 +399,7 @@ message RefreshPeerParameter { required ServerName target_server = 3; } -message ModifyPeerStateData { +message PeerProcedureStateData { required string peer_id = 1; } http://git-wip-us.apache.org/repos/asf/hbase/blob/1c4938da/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java new file mode 100644 index 000..0ad8a63 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.replication; + +import java.io.IOException; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface; +import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; +import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerProcedureStateData; + +/** + * The base class for all replication peer related procedure. + */ +@InterfaceAudience.Private +public abstract class AbstractPeerProcedure +extends StateMachineProcedure implements PeerProcedureInterface { + + protected String peerId; + + private volatile boolean locked; + + // used to keep compatible with old client where we can only returns after updateStorage. + protected ProcedurePrepareLatch latch; + + protected AbstractPeerProcedure() { + } + + protected AbstractPeerProcedure(String peerId) { +this.peerId = peerId; +this.latch = ProcedurePrepareLatch.createLatch(2, 0); + } + + public ProcedurePrepareLatch getLatch() { +return latch; + } + + @Override + public String getPeerId() { +return peerId; + } + + @Override + protected LockState acquireLock(MasterProcedureEnv env) { +if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) { + return LockState.LOCK_EVENT_WAIT; +} +locked = true; +return LockState.LOCK_ACQUIRED; + } + + @Override + protected void releaseLock(MasterProcedureEnv env) { +locked = false; +env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId); + } + + @Override + protected boolean holdLock(MasterProcedureEnv env) { +return true; + } + + @Override + protecte
[09/50] [abbrv] hbase git commit: HBASE-20065 Addendum remove wrong comment
HBASE-20065 Addendum remove wrong comment Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10270e36 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10270e36 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10270e36 Branch: refs/heads/HBASE-19397-branch-2 Commit: 10270e36c9bd8571135ca7c8692a9e1ece0a6a45 Parents: 8b3ae58 Author: zhangduo Authored: Mon Feb 26 09:48:41 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 09:48:55 2018 +0800 -- .../src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java| 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/10270e36/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 803f183..e779054 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -1535,7 +1535,6 @@ public class MetaTableAccessor { RegionInfo regionA, RegionInfo regionB, ServerName sn, int regionReplication) throws IOException { try (Table meta = getMetaHTable(connection)) { - // use the maximum of what master passed us vs local time. long time = EnvironmentEdgeManager.currentTime(); // Put for parent
[02/50] [abbrv] hbase git commit: HBASE-20048 Revert serial replication feature
HBASE-20048 Revert serial replication feature Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4ddfdaff Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4ddfdaff Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4ddfdaff Branch: refs/heads/HBASE-19397-branch-2 Commit: 4ddfdaffdcf83d50aac26fcff5e1ac4ecd575335 Parents: 8a22e41 Author: zhangduo Authored: Fri Feb 23 08:51:37 2018 +0800 Committer: zhangduo Committed: Fri Feb 23 13:54:10 2018 +0800 -- .../apache/hadoop/hbase/HTableDescriptor.java | 8 - .../apache/hadoop/hbase/MetaTableAccessor.java | 296 ++ .../hadoop/hbase/client/TableDescriptor.java| 9 +- .../hbase/client/TableDescriptorBuilder.java| 17 +- .../client/replication/ReplicationAdmin.java| 10 +- .../client/TestTableDescriptorBuilder.java | 25 +- .../org/apache/hadoop/hbase/HConstants.java | 33 -- .../src/main/resources/hbase-default.xml| 13 - hbase-protocol/src/main/protobuf/WAL.proto | 1 - .../org/apache/hadoop/hbase/master/HMaster.java | 8 +- .../master/assignment/RegionStateStore.java | 23 +- .../master/cleaner/ReplicationMetaCleaner.java | 191 - .../hbase/regionserver/HRegionServer.java | 77 ++-- .../hbase/regionserver/wal/FSWALEntry.java | 1 + .../RecoveredReplicationSourceShipper.java | 3 +- .../regionserver/ReplicationSourceManager.java | 121 +- .../regionserver/ReplicationSourceShipper.java | 76 +--- .../ReplicationSourceWALReader.java | 52 +-- .../hbase/snapshot/RestoreSnapshotHelper.java | 2 +- .../hadoop/hbase/util/FSTableDescriptors.java | 30 -- .../java/org/apache/hadoop/hbase/wal/WAL.java | 19 +- .../hadoop/hbase/TestMetaTableAccessor.java | 10 +- .../regionserver/TestRegionServerMetrics.java | 2 +- .../replication/TestSerialReplication.java | 400 --- .../regionserver/TestGlobalThrottler.java | 2 +- src/main/asciidoc/_chapters/ops_mgt.adoc| 41 +- 26 files changed, 87 insertions(+), 1383 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index c9807c3..e512b2c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -538,14 +538,6 @@ public class HTableDescriptor implements TableDescriptor, Comparablehttp://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index dad9aef..7d00f92 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -17,13 +17,14 @@ */ package org.apache.hadoop.hbase; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Closeable; import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,7 +34,6 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell.Type; import org.apache.hadoop.hbase.client.Connection; @@ -71,9 +71,8 @@ import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.Nullable; /** * Read/write operations on region and assignment information store in @@ -123,34 +122,14 @@ public class MetaTableAccessor { * region is the result of a merge * info:mergeB => contains a serialized HRI for the second parent region if the * region is the result of a merge + * * The actual layout of meta should be en
[19/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b75cffe0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b75cffe0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b75cffe0 Branch: refs/heads/HBASE-19397-branch-2 Commit: b75cffe07cbfa6eb1d80c771819ab75ff83485a6 Parents: d8f09f1 Author: zhangduo Authored: Mon Dec 25 18:49:56 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hbase/replication/ReplicationFactory.java | 19 +- .../replication/ReplicationPeersZKImpl.java | 21 +- .../replication/ReplicationQueueStorage.java| 26 +- .../replication/ReplicationQueuesClient.java| 93 - .../ReplicationQueuesClientArguments.java | 40 -- .../ReplicationQueuesClientZKImpl.java | 181 - .../replication/ZKReplicationQueueStorage.java | 90 - .../replication/TestReplicationStateBasic.java | 378 +++ .../replication/TestReplicationStateZKImpl.java | 153 .../TestZKReplicationQueueStorage.java | 74 .../cleaner/ReplicationZKNodeCleaner.java | 71 ++-- .../cleaner/ReplicationZKNodeCleanerChore.java | 5 +- .../replication/ReplicationPeerManager.java | 31 +- .../master/ReplicationHFileCleaner.java | 109 ++ .../master/ReplicationLogCleaner.java | 44 +-- .../regionserver/DumpReplicationQueues.java | 78 ++-- .../hbase/util/hbck/ReplicationChecker.java | 14 +- .../client/TestAsyncReplicationAdminApi.java| 31 +- .../replication/TestReplicationAdmin.java | 2 + .../hbase/master/cleaner/TestLogsCleaner.java | 81 ++-- .../cleaner/TestReplicationHFileCleaner.java| 29 -- .../cleaner/TestReplicationZKNodeCleaner.java | 12 +- .../replication/TestReplicationStateBasic.java | 378 --- .../replication/TestReplicationStateZKImpl.java | 232 .../TestReplicationSourceManagerZkImpl.java | 41 -- 25 files changed, 890 insertions(+), 1343 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java index 9f4ad18..6c1c213 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java @@ -1,5 +1,4 @@ -/* - * +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,20 +36,14 @@ public class ReplicationFactory { args); } - public static ReplicationQueuesClient - getReplicationQueuesClient(ReplicationQueuesClientArguments args) throws Exception { -return (ReplicationQueuesClient) ConstructorUtils -.invokeConstructor(ReplicationQueuesClientZKImpl.class, args); - } - - public static ReplicationPeers getReplicationPeers(final ZKWatcher zk, Configuration conf, - Abortable abortable) { + public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf, + Abortable abortable) { return getReplicationPeers(zk, conf, null, abortable); } - public static ReplicationPeers getReplicationPeers(final ZKWatcher zk, Configuration conf, - final ReplicationQueuesClient queuesClient, Abortable abortable) { -return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable); + public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf, + ReplicationQueueStorage queueStorage, Abortable abortable) { +return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable); } public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper, http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java index 419e289..4e5f757 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/h
[25/50] [abbrv] hbase git commit: HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure
HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29af06ab Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29af06ab Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29af06ab Branch: refs/heads/HBASE-19397-branch-2 Commit: 29af06ab4b4d3ca7e6d7ba1597d176b97a9f863a Parents: febfe26 Author: Guanghao Zhang Authored: Tue Dec 19 15:50:57 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../org/apache/hadoop/hbase/client/Admin.java | 87 ++- .../apache/hadoop/hbase/client/HBaseAdmin.java | 149 ++- .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 82 +- 3 files changed, 238 insertions(+), 80 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/29af06ab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 40dac2f..b8546fa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2473,7 +2473,7 @@ public interface Admin extends Abortable, Closeable { /** * Add a new replication peer for replicating data to slave cluster. * @param peerId a short name that identifies the peer - * @param peerConfig configuration for the replication slave cluster + * @param peerConfig configuration for the replication peer * @throws IOException if a remote or network exception occurs */ default void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig) @@ -2484,7 +2484,7 @@ public interface Admin extends Abortable, Closeable { /** * Add a new replication peer for replicating data to slave cluster. * @param peerId a short name that identifies the peer - * @param peerConfig configuration for the replication slave cluster + * @param peerConfig configuration for the replication peer * @param enabled peer state, true if ENABLED and false if DISABLED * @throws IOException if a remote or network exception occurs */ @@ -2492,6 +2492,37 @@ public interface Admin extends Abortable, Closeable { throws IOException; /** + * Add a new replication peer but does not block and wait for it. + * + * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw + * ExecutionException if there was an error while executing the operation or TimeoutException in + * case the wait timeout was not long enough to allow the operation to complete. + * @param peerId a short name that identifies the peer + * @param peerConfig configuration for the replication peer + * @return the result of the async operation + * @throws IOException IOException if a remote or network exception occurs + */ + default Future addReplicationPeerAsync(String peerId, ReplicationPeerConfig peerConfig) + throws IOException { +return addReplicationPeerAsync(peerId, peerConfig, true); + } + + /** + * Add a new replication peer but does not block and wait for it. + * + * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw + * ExecutionException if there was an error while executing the operation or TimeoutException in + * case the wait timeout was not long enough to allow the operation to complete. + * @param peerId a short name that identifies the peer + * @param peerConfig configuration for the replication peer + * @param enabled peer state, true if ENABLED and false if DISABLED + * @return the result of the async operation + * @throws IOException IOException if a remote or network exception occurs + */ + Future addReplicationPeerAsync(String peerId, ReplicationPeerConfig peerConfig, + boolean enabled) throws IOException; + + /** * Remove a peer and stop the replication. * @param peerId a short name that identifies the peer * @throws IOException if a remote or network exception occurs @@ -2499,6 +2530,18 @@ public interface Admin extends Abortable, Closeable { void removeReplicationPeer(String peerId) throws IOException; /** + * Remove a replication peer but does not block and wait for it. + * + * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw + * ExecutionException if there was an error while executing the operation or TimeoutException in + * case the wait timeout was not long enough to allow the operation to complete. + * @param peerId a shor
[38/50] [abbrv] hbase git commit: HBASE-19634 Add permission check for executeProcedures in AccessController
HBASE-19634 Add permission check for executeProcedures in AccessController Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f42fc70b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f42fc70b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f42fc70b Branch: refs/heads/HBASE-19397-branch-2 Commit: f42fc70b13b8f4009eb4cac6d745eb36bc22f770 Parents: d448c01 Author: zhangduo Authored: Thu Jan 4 16:18:21 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:03 2018 +0800 -- .../hbase/coprocessor/RegionServerObserver.java | 14 ++ .../hbase/regionserver/RSRpcServices.java | 52 +++- .../RegionServerCoprocessorHost.java| 18 +++ .../hbase/security/access/AccessController.java | 30 ++- .../hadoop/hbase/TestJMXConnectorServer.java| 7 +++ .../security/access/TestAccessController.java | 18 +-- 6 files changed, 100 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/f42fc70b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java index c1af3fb..5b751df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java @@ -126,4 +126,18 @@ public interface RegionServerObserver { default void postClearCompactionQueues( final ObserverContext ctx) throws IOException {} + + /** + * This will be called before executing procedures + * @param ctx the environment to interact with the framework and region server. + */ + default void preExecuteProcedures(ObserverContext ctx) + throws IOException {} + + /** + * This will be called after executing procedures + * @param ctx the environment to interact with the framework and region server. + */ + default void postExecuteProcedures(ObserverContext ctx) + throws IOException {} } http://git-wip-us.apache.org/repos/asf/hbase/blob/f42fc70b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 74f056b..88ce346 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -3508,36 +3508,40 @@ public class RSRpcServices implements HBaseRPCErrorHandler, } @Override + @QosPriority(priority = HConstants.ADMIN_QOS) public ExecuteProceduresResponse executeProcedures(RpcController controller, ExecuteProceduresRequest request) throws ServiceException { -if (request.getOpenRegionCount() > 0) { - for (OpenRegionRequest req : request.getOpenRegionList()) { -openRegion(controller, req); +try { + checkOpen(); + regionServer.getRegionServerCoprocessorHost().preExecuteProcedures(); + if (request.getOpenRegionCount() > 0) { +for (OpenRegionRequest req : request.getOpenRegionList()) { + openRegion(controller, req); +} } -} -if (request.getCloseRegionCount() > 0) { - for (CloseRegionRequest req : request.getCloseRegionList()) { -closeRegion(controller, req); + if (request.getCloseRegionCount() > 0) { +for (CloseRegionRequest req : request.getCloseRegionList()) { + closeRegion(controller, req); +} } -} -if (request.getProcCount() > 0) { - for (RemoteProcedureRequest req : request.getProcList()) { -RSProcedureCallable callable; -try { - callable = - Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance(); -} catch (Exception e) { - // here we just ignore the error as this should not happen and we do not provide a general - // way to report errors for all types of remote procedure. The procedure will hang at - // master side but after you solve the problem and restart master it will be executed - // again and pass. - LOG.warn("create procedure of type " + req.getProcClass() + " failed, give up", e); - continue; + if (request.getProcCount() > 0) { +for (RemoteProcedureRequest req : request.getProcList()) { +
[04/50] [abbrv] hbase git commit: Revert "HBASE-19835 Use explicit casting to avoid side effects"
Revert "HBASE-19835 Use explicit casting to avoid side effects" This reverts commit bc080e75000aa16ee05d42033ec5e4d984985205. Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java patch reverted changes that happened in parallel without explanation. see jira. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a66fa85a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a66fa85a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a66fa85a Branch: refs/heads/HBASE-19397-branch-2 Commit: a66fa85a1650808658db9556dbdf1fac5745c9b2 Parents: d6d1e49 Author: Sean Busbey Authored: Fri Feb 23 19:46:12 2018 -0600 Committer: Sean Busbey Committed: Fri Feb 23 23:04:34 2018 -0600 -- .../org/apache/hadoop/hbase/client/Append.java | 2 +- .../apache/hadoop/hbase/client/Mutation.java| 6 +- .../hadoop/hbase/IndividualBytesFieldCell.java | 2 +- .../apache/hadoop/hbase/PrivateCellUtil.java| 4 +- .../hbase/regionserver/CompactionPipeline.java | 2 +- .../hadoop/hbase/regionserver/HRegion.java | 4 +- .../hadoop/hbase/regionserver/HStore.java | 94 +++- .../hbase/regionserver/MemStoreCompactor.java | 2 +- .../MultiVersionConcurrencyControl.java | 2 +- .../hadoop/hbase/regionserver/ScanInfo.java | 2 +- .../hadoop/hbase/regionserver/wal/FSHLog.java | 2 +- .../apache/hadoop/hbase/wal/WALSplitter.java| 2 +- 12 files changed, 67 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java index 042209c..61474b7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; @InterfaceAudience.Public public class Append extends Mutation { private static final Logger LOG = LoggerFactory.getLogger(Append.class); - private static final long HEAP_OVERHEAD = (long)ClassSize.REFERENCE + ClassSize.TIMERANGE; + private static final long HEAP_OVERHEAD = ClassSize.REFERENCE + ClassSize.TIMERANGE; private TimeRange tr = new TimeRange(); /** http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java index 09000ac..1338fff 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java @@ -66,7 +66,7 @@ public abstract class Mutation extends OperationWithAttributes implements Row, C HeapSize { public static final long MUTATION_OVERHEAD = ClassSize.align( // This - (long)ClassSize.OBJECT + + ClassSize.OBJECT + // row + OperationWithAttributes.attributes 2 * ClassSize.REFERENCE + // Timestamp @@ -791,10 +791,10 @@ public abstract class Mutation extends OperationWithAttributes implements Row, C private static final class CellWrapper implements ExtendedCell { private static final long FIXED_OVERHEAD = ClassSize.align( - (long)ClassSize.OBJECT // object header + ClassSize.OBJECT // object header + KeyValue.TIMESTAMP_SIZE // timestamp + Bytes.SIZEOF_LONG // sequence id -+ 1L * ClassSize.REFERENCE); // references to cell ++ 1 * ClassSize.REFERENCE); // references to cell private final Cell cell; private long sequenceId; private long timestamp; http://git-wip-us.apache.org/repos/asf/hbase/blob/a66fa85a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java index 6a95a7c..7093b4b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java @@ -27,7 +27,7 @@ import org.apache.yetus.audience.Inte
[29/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fef99645 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fef99645 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fef99645 Branch: refs/heads/HBASE-19397-branch-2 Commit: fef996454fc5e44878cf972f744920358d20ebf1 Parents: df6fc97 Author: huzheng Authored: Tue Dec 26 16:46:10 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:24 2018 +0800 -- .../replication/ReplicationPeerConfigUtil.java | 10 +- .../replication/VerifyReplication.java | 9 +- .../hbase/replication/ReplicationFactory.java | 10 +- .../hbase/replication/ReplicationPeerImpl.java | 60 +- .../replication/ReplicationPeerStorage.java | 3 +- .../hbase/replication/ReplicationPeers.java | 238 .../replication/ReplicationPeersZKImpl.java | 552 --- .../replication/ZKReplicationPeerStorage.java | 12 +- .../replication/ZKReplicationStorageBase.java | 3 +- .../replication/TestReplicationStateBasic.java | 125 ++--- .../replication/TestReplicationStateZKImpl.java | 2 +- .../TestZKReplicationPeerStorage.java | 12 +- .../cleaner/ReplicationZKNodeCleaner.java | 57 +- .../replication/ReplicationPeerManager.java | 6 +- .../regionserver/DumpReplicationQueues.java | 2 +- .../regionserver/PeerProcedureHandlerImpl.java | 49 +- .../replication/regionserver/Replication.java | 2 +- .../regionserver/ReplicationSource.java | 7 +- .../regionserver/ReplicationSourceManager.java | 44 +- .../cleaner/TestReplicationHFileCleaner.java| 7 +- .../replication/TestMultiSlaveReplication.java | 2 - .../TestReplicationTrackerZKImpl.java | 26 +- .../TestReplicationSourceManager.java | 17 +- .../hadoop/hbase/HBaseZKTestingUtility.java | 3 +- 24 files changed, 307 insertions(+), 951 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java index 022bf64..a234a9b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java @@ -247,22 +247,22 @@ public final class ReplicationPeerConfigUtil { public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes) throws DeserializationException { if (ProtobufUtil.isPBMagicPrefix(bytes)) { - int pblen = ProtobufUtil.lengthOfPBMagic(); + int pbLen = ProtobufUtil.lengthOfPBMagic(); ReplicationProtos.ReplicationPeer.Builder builder = ReplicationProtos.ReplicationPeer.newBuilder(); ReplicationProtos.ReplicationPeer peer; try { -ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen); +ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen); peer = builder.build(); } catch (IOException e) { throw new DeserializationException(e); } return convert(peer); } else { - if (bytes.length > 0) { -return ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build(); + if (bytes == null || bytes.length <= 0) { +throw new DeserializationException("Bytes to deserialize should not be empty."); } - return ReplicationPeerConfig.newBuilder().setClusterKey("").build(); + return ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/fef99645/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index 09d4b4b..f0070f0 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -339,15 +339,10 @@ public class VerifyReplication extends Configured implements Tool { @Override public boolea
[24/50] [abbrv] hbase git commit: HBASE-19564 Procedure id is missing in the response of peer related operations
HBASE-19564 Procedure id is missing in the response of peer related operations Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50513f84 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50513f84 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50513f84 Branch: refs/heads/HBASE-19397-branch-2 Commit: 50513f845e1470d89bd84621b371579d2f0616dc Parents: 29af06a Author: zhangduo Authored: Wed Dec 20 20:57:37 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hadoop/hbase/master/MasterRpcServices.java | 24 ++-- .../master/replication/ModifyPeerProcedure.java | 4 +--- 2 files changed, 13 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/50513f84/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 3c24e4d..4622a00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1866,10 +1866,10 @@ public class MasterRpcServices extends RSRpcServices public AddReplicationPeerResponse addReplicationPeer(RpcController controller, AddReplicationPeerRequest request) throws ServiceException { try { - master.addReplicationPeer(request.getPeerId(), -ReplicationPeerConfigUtil.convert(request.getPeerConfig()), request.getPeerState() -.getState().equals(ReplicationState.State.ENABLED)); - return AddReplicationPeerResponse.newBuilder().build(); + long procId = master.addReplicationPeer(request.getPeerId(), +ReplicationPeerConfigUtil.convert(request.getPeerConfig()), + request.getPeerState().getState().equals(ReplicationState.State.ENABLED)); + return AddReplicationPeerResponse.newBuilder().setProcId(procId).build(); } catch (ReplicationException | IOException e) { throw new ServiceException(e); } @@ -1879,8 +1879,8 @@ public class MasterRpcServices extends RSRpcServices public RemoveReplicationPeerResponse removeReplicationPeer(RpcController controller, RemoveReplicationPeerRequest request) throws ServiceException { try { - master.removeReplicationPeer(request.getPeerId()); - return RemoveReplicationPeerResponse.newBuilder().build(); + long procId = master.removeReplicationPeer(request.getPeerId()); + return RemoveReplicationPeerResponse.newBuilder().setProcId(procId).build(); } catch (ReplicationException | IOException e) { throw new ServiceException(e); } @@ -1890,8 +1890,8 @@ public class MasterRpcServices extends RSRpcServices public EnableReplicationPeerResponse enableReplicationPeer(RpcController controller, EnableReplicationPeerRequest request) throws ServiceException { try { - master.enableReplicationPeer(request.getPeerId()); - return EnableReplicationPeerResponse.newBuilder().build(); + long procId = master.enableReplicationPeer(request.getPeerId()); + return EnableReplicationPeerResponse.newBuilder().setProcId(procId).build(); } catch (ReplicationException | IOException e) { throw new ServiceException(e); } @@ -1901,8 +1901,8 @@ public class MasterRpcServices extends RSRpcServices public DisableReplicationPeerResponse disableReplicationPeer(RpcController controller, DisableReplicationPeerRequest request) throws ServiceException { try { - master.disableReplicationPeer(request.getPeerId()); - return DisableReplicationPeerResponse.newBuilder().build(); + long procId = master.disableReplicationPeer(request.getPeerId()); + return DisableReplicationPeerResponse.newBuilder().setProcId(procId).build(); } catch (ReplicationException | IOException e) { throw new ServiceException(e); } @@ -1928,9 +1928,9 @@ public class MasterRpcServices extends RSRpcServices public UpdateReplicationPeerConfigResponse updateReplicationPeerConfig(RpcController controller, UpdateReplicationPeerConfigRequest request) throws ServiceException { try { - master.updateReplicationPeerConfig(request.getPeerId(), + long procId = master.updateReplicationPeerConfig(request.getPeerId(), ReplicationPeerConfigUtil.convert(request.getPeerConfig())); - return UpdateReplicationPeerConfigResponse.newBuilder().build(); + return UpdateReplicationPeerConfigResponse.newBuilder().setProcId(procId).build(); } catch (ReplicationException | IOException e) { thr
[23/50] [abbrv] hbase git commit: HBASE-19592 Add UTs to test retry on update zk failure
HBASE-19592 Add UTs to test retry on update zk failure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b13faf0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b13faf0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b13faf0 Branch: refs/heads/HBASE-19397-branch-2 Commit: 3b13faf09557074c9ff52714f20cea31ffd483b3 Parents: b3be773 Author: zhangduo Authored: Tue Dec 26 20:39:00 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../replication/ReplicationPeerManager.java | 5 +- .../TestReplicationProcedureRetry.java | 200 +++ 2 files changed, 202 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3b13faf0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java index b78cbce..f4ccce8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java @@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience; * Used to add/remove a replication peer. */ @InterfaceAudience.Private -public final class ReplicationPeerManager { +public class ReplicationPeerManager { private final ReplicationPeerStorage peerStorage; @@ -61,8 +61,7 @@ public final class ReplicationPeerManager { private final ConcurrentMap peers; - private ReplicationPeerManager(ReplicationPeerStorage peerStorage, - ReplicationQueueStorage queueStorage, + ReplicationPeerManager(ReplicationPeerStorage peerStorage, ReplicationQueueStorage queueStorage, ConcurrentMap peers) { this.peerStorage = peerStorage; this.queueStorage = queueStorage; http://git-wip-us.apache.org/repos/asf/hbase/blob/3b13faf0/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java new file mode 100644 index 000..ab35b46 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.invocation.InvocationOnMock; + +/** + * All the modification method will fail once in the test and should finally succeed. + */ +@Category({ Replic
[27/50] [abbrv] hbase git commit: HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
HBASE-19635 Introduce a thread at RS side to call reportProcedureDone Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df6fc97f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df6fc97f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df6fc97f Branch: refs/heads/HBASE-19397-branch-2 Commit: df6fc97f5f60db420af7f7ce63461e40b8bec3f8 Parents: 6bbc607 Author: zhangduo Authored: Wed Dec 27 20:13:42 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:24 2018 +0800 -- .../src/main/protobuf/RegionServerStatus.proto | 5 +- .../hadoop/hbase/master/MasterRpcServices.java | 15 ++- .../hbase/regionserver/HRegionServer.java | 72 .../RemoteProcedureResultReporter.java | 111 +++ .../handler/RSProcedureHandler.java | 2 +- 5 files changed, 149 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto index 4f75941..3f836cd 100644 --- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto @@ -146,7 +146,7 @@ message RegionSpaceUseReportRequest { message RegionSpaceUseReportResponse { } -message ReportProcedureDoneRequest { +message RemoteProcedureResult { required uint64 proc_id = 1; enum Status { SUCCESS = 1; @@ -155,6 +155,9 @@ message ReportProcedureDoneRequest { required Status status = 2; optional ForeignExceptionMessage error = 3; } +message ReportProcedureDoneRequest { + repeated RemoteProcedureResult result = 1; +} message ReportProcedureDoneResponse { } http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 4622a00..b4f0faf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -266,6 +266,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest; @@ -2234,12 +2235,14 @@ public class MasterRpcServices extends RSRpcServices @Override public ReportProcedureDoneResponse reportProcedureDone(RpcController controller, ReportProcedureDoneRequest request) throws ServiceException { -if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) { - master.remoteProcedureCompleted(request.getProcId()); -} else { - master.remoteProcedureFailed(request.getProcId(), -RemoteProcedureException.fromProto(request.getError())); -} +request.getResultList().forEach(result -> { + if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) { +master.remoteProcedureCompleted(result.getProcId()); + } else { +master.remoteProcedureFailed(result.getProcId(), + RemoteProcedureException.fromProto(result.getError())); + } +}); return ReportProcedureDoneResponse.getDefaultInstance(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/df6fc97f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index c95ac37..81febc0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/region
[39/50] [abbrv] hbase git commit: HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/85e85bea Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/85e85bea Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/85e85bea Branch: refs/heads/HBASE-19397-branch-2 Commit: 85e85bea03e57b766d251874740cb4cdbc981abd Parents: 718e664 Author: huzheng Authored: Fri Dec 29 15:55:28 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:03 2018 +0800 -- .../hbase/replication/ReplicationFactory.java | 5 +- .../replication/ReplicationStateZKBase.java | 159 --- .../replication/ReplicationTrackerZKImpl.java | 21 +-- .../replication/ZKReplicationPeerStorage.java | 24 ++- .../replication/ZKReplicationStorageBase.java | 13 +- .../org/apache/hadoop/hbase/master/HMaster.java | 4 +- .../master/ReplicationPeerConfigUpgrader.java | 128 +++ .../regionserver/DumpReplicationQueues.java | 18 +-- .../replication/regionserver/Replication.java | 3 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +- .../TestReplicationTrackerZKImpl.java | 3 +- .../replication/master/TestTableCFsUpdater.java | 41 ++--- .../TestReplicationSourceManager.java | 6 +- 13 files changed, 136 insertions(+), 292 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/85e85bea/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java index 6c66aff..2a970ba 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java @@ -33,9 +33,8 @@ public class ReplicationFactory { return new ReplicationPeers(zk, conf); } - public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper, - final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable, + public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) { -return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf, abortable, stopper); +return new ReplicationTrackerZKImpl(zookeeper, abortable, stopper); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/85e85bea/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java deleted file mode 100644 index a48683e..000 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos; -import org.apache.hadoop.hbase.zookeeper.ZKConfig; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.apache.hadoop.hbase.zookeeper.ZNodePaths; -import org.apache.yetus.audi
[44/50] [abbrv] hbase git commit: HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b980c46 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b980c46 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b980c46 Branch: refs/heads/HBASE-19397-branch-2 Commit: 0b980c4603366e62dfbb1e56a17ac39693ddbbc3 Parents: f8b58c3 Author: huzheng Authored: Wed Jan 10 15:00:30 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../main/java/org/apache/hadoop/hbase/master/HMaster.java | 9 - 1 file changed, 4 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/0b980c46/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 45058a1..f55d065 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -71,6 +70,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.PleaseHoldException; +import org.apache.hadoop.hbase.ReplicationPeerNotFoundException; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerMetricsBuilder; import org.apache.hadoop.hbase.ServerName; @@ -3409,13 +3409,12 @@ public class HMaster extends HRegionServer implements MasterServices { cpHost.preGetReplicationPeerConfig(peerId); } LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId); -Optional peerConfig = - this.replicationPeerManager.getPeerConfig(peerId); - +ReplicationPeerConfig peerConfig = this.replicationPeerManager.getPeerConfig(peerId) +.orElseThrow(() -> new ReplicationPeerNotFoundException(peerId)); if (cpHost != null) { cpHost.postGetReplicationPeerConfig(peerId); } -return peerConfig.orElse(null); +return peerConfig; } @Override
[45/50] [abbrv] hbase git commit: HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/03984c67 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/03984c67 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/03984c67 Branch: refs/heads/HBASE-19397-branch-2 Commit: 03984c6787e98fe2bf96a7817b9318fdbd6c962f Parents: 8cac522 Author: Guanghao Zhang Authored: Fri Jan 5 15:39:06 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../procedure/MasterProcedureScheduler.java | 23 1 file changed, 19 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/03984c67/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java index b18dd6c..eab06a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java @@ -290,7 +290,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { @Override public void completionCleanup(final Procedure proc) { if (proc instanceof TableProcedureInterface) { - TableProcedureInterface iProcTable = (TableProcedureInterface)proc; + TableProcedureInterface iProcTable = (TableProcedureInterface) proc; boolean tableDeleted; if (proc.hasException()) { Exception procEx = proc.getException().unwrapRemoteException(); @@ -311,9 +311,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { } } else if (proc instanceof PeerProcedureInterface) { PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc; - if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) { -removePeerQueue(iProcPeer.getPeerId()); - } + tryCleanupPeerQueue(iProcPeer.getPeerId(), proc); } else { // No cleanup for ServerProcedureInterface types, yet. return; @@ -402,6 +400,23 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { locking.removePeerLock(peerId); } + private void tryCleanupPeerQueue(String peerId, Procedure procedure) { +schedLock(); +try { + PeerQueue queue = AvlTree.get(peerMap, peerId, PEER_QUEUE_KEY_COMPARATOR); + if (queue == null) { +return; + } + + final LockAndQueue lock = locking.getPeerLock(peerId); + if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) { +removeFromRunQueue(peerRunQueue, queue); +removePeerQueue(peerId); + } +} finally { + schedUnlock(); +} + } private static boolean isPeerProcedure(Procedure proc) { return proc instanceof PeerProcedureInterface;
[36/50] [abbrv] hbase git commit: HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore
HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/718e664b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/718e664b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/718e664b Branch: refs/heads/HBASE-19397-branch-2 Commit: 718e664b707745d18226eae90796d7c0965c7534 Parents: 1d7960c Author: zhangduo Authored: Wed Jan 3 09:39:44 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:03 2018 +0800 -- .../replication/VerifyReplication.java | 6 +- .../hbase/replication/ReplicationPeers.java | 26 +-- .../hbase/replication/ReplicationUtils.java | 38 .../replication/TestReplicationStateBasic.java | 2 +- .../org/apache/hadoop/hbase/master/HMaster.java | 13 -- .../cleaner/ReplicationZKNodeCleaner.java | 192 --- .../cleaner/ReplicationZKNodeCleanerChore.java | 54 -- .../replication/ReplicationPeerManager.java | 18 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 11 +- .../hbase/util/hbck/ReplicationChecker.java | 109 +++ .../cleaner/TestReplicationZKNodeCleaner.java | 115 --- .../hbase/util/TestHBaseFsckReplication.java| 101 ++ .../hadoop/hbase/util/hbck/HbckTestingUtil.java | 6 +- 13 files changed, 226 insertions(+), 465 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/718e664b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index fe45762..fac4875 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; +import org.apache.hadoop.hbase.replication.ReplicationUtils; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -345,10 +345,10 @@ public class VerifyReplication extends Configured implements Tool { } }); ReplicationPeerStorage storage = - ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf); +ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf); ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId); return Pair.newPair(peerConfig, -ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf)); +ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf)); } catch (ReplicationException e) { throw new IOException("An error occurred while trying to connect to the remove peer cluster", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/718e664b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java index 45940a5..fcbc350 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.hbase.replication; -import java.io.IOException; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CompoundConfiguration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.yetus.audience.InterfaceAudience; @@ -106,25 +103,6 @@ public class ReplicationPeers { return Collections.unmodifiableSet(peerCache.keySet()); } - public
[34/50] [abbrv] hbase git commit: HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a75ee53d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a75ee53d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a75ee53d Branch: refs/heads/HBASE-19397-branch-2 Commit: a75ee53d8440d9ca71a20587fb34c558c97ba6f9 Parents: 8a762b6 Author: zhangduo Authored: Tue Jan 2 16:13:55 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:58 2018 +0800 -- .../regionserver/PeerProcedureHandlerImpl.java | 41 ++-- 1 file changed, 29 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a75ee53d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java index 1efe180..c09c6a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java @@ -19,10 +19,10 @@ package org.apache.hadoop.hbase.replication.regionserver; import java.io.IOException; -import java.util.concurrent.locks.ReentrantLock; - +import java.util.concurrent.locks.Lock; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState; +import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +32,7 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler { private static final Logger LOG = LoggerFactory.getLogger(PeerProcedureHandlerImpl.class); private final ReplicationSourceManager replicationSourceManager; - private final ReentrantLock peersLock = new ReentrantLock(); + private final KeyLocker peersLock = new KeyLocker<>(); public PeerProcedureHandlerImpl(ReplicationSourceManager replicationSourceManager) { this.replicationSourceManager = replicationSourceManager; @@ -40,40 +40,57 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler { @Override public void addPeer(String peerId) throws ReplicationException, IOException { -peersLock.lock(); +Lock peerLock = peersLock.acquireLock(peerId); try { replicationSourceManager.addPeer(peerId); } finally { - peersLock.unlock(); + peerLock.unlock(); } } @Override public void removePeer(String peerId) throws ReplicationException, IOException { -peersLock.lock(); +Lock peerLock = peersLock.acquireLock(peerId); try { if (replicationSourceManager.getReplicationPeers().getPeer(peerId) != null) { replicationSourceManager.removePeer(peerId); } } finally { - peersLock.unlock(); + peerLock.unlock(); } } @Override public void disablePeer(String peerId) throws ReplicationException, IOException { -PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId); -LOG.info("disable replication peer, id: " + peerId + ", new state: " + newState); +PeerState newState; +Lock peerLock = peersLock.acquireLock(peerId); +try { + newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId); +} finally { + peerLock.unlock(); +} +LOG.info("disable replication peer, id: {}, new state: {}", peerId, newState); } @Override public void enablePeer(String peerId) throws ReplicationException, IOException { -PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId); -LOG.info("enable replication peer, id: " + peerId + ", new state: " + newState); +PeerState newState; +Lock peerLock = peersLock.acquireLock(peerId); +try { + newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId); +} finally { + peerLock.unlock(); +} +LOG.info("enable replication peer, id: {}, new state: {}", peerId, newState); } @Override public void updatePeerConfig(String peerId) throws ReplicationException, IOException { -replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId); +Lock peerLock = peersLock.acquireLock(peerId); +try { + replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId); +} finally { + peerLock.unlock(
[43/50] [abbrv] hbase git commit: HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed
HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c26916dd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c26916dd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c26916dd Branch: refs/heads/HBASE-19397-branch-2 Commit: c26916ddb78c66563974113d649e4af430627394 Parents: 0b980c4 Author: Guanghao Zhang Authored: Fri Jan 12 22:04:38 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../replication/ReplicationPeerManager.java | 28 +--- 1 file changed, 19 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c26916dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java index 696b2d7..19fc7f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java @@ -132,20 +132,19 @@ public class ReplicationPeerManager { checkPeerConfig(peerConfig); ReplicationPeerDescription desc = checkPeerExists(peerId); ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig(); -if (!StringUtils.isBlank(peerConfig.getClusterKey()) && - !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) { +if (!isStringEquals(peerConfig.getClusterKey(), oldPeerConfig.getClusterKey())) { throw new DoNotRetryIOException( "Changing the cluster key on an existing peer is not allowed. Existing key '" + -oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does not match new key '" + -peerConfig.getClusterKey() + "'"); + oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does not match new key '" + + peerConfig.getClusterKey() + "'"); } -if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) && - !peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl())) { +if (!isStringEquals(peerConfig.getReplicationEndpointImpl(), + oldPeerConfig.getReplicationEndpointImpl())) { throw new DoNotRetryIOException("Changing the replication endpoint implementation class " + -"on an existing peer is not allowed. Existing class '" + -oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId + -" does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'"); + "on an existing peer is not allowed. Existing class '" + + oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId + + " does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'"); } } @@ -341,4 +340,15 @@ public class ReplicationPeerManager { return new ReplicationPeerManager(peerStorage, ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers); } + + /** + * For replication peer cluster key or endpoint class, null and empty string is same. So here + * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly. + */ + private boolean isStringEquals(String s1, String s2) { +if (StringUtils.isBlank(s1)) { + return StringUtils.isBlank(s2); +} +return s1.equals(s2); + } }
[03/50] [abbrv] hbase git commit: HBASE-20054 removes the indefinite retry introduced by HBASE-18282
HBASE-20054 removes the indefinite retry introduced by HBASE-18282 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d6d1e49a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d6d1e49a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d6d1e49a Branch: refs/heads/HBASE-19397-branch-2 Commit: d6d1e49a7561ac1b13d965bf74436919de625f1d Parents: 4ddfdaf Author: tedyu Authored: Fri Feb 23 19:14:04 2018 -0800 Committer: tedyu Committed: Fri Feb 23 19:14:04 2018 -0800 -- .../regionserver/ReplicationSourceManager.java | 19 +-- 1 file changed, 5 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d6d1e49a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index 2cf3a82..bfd338d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -782,20 +782,11 @@ public class ReplicationSourceManager implements ReplicationListener { @Override public void run() { List currentReplicators = null; - while (currentReplicators == null) { -try { - currentReplicators = replicationQueues.getListOfReplicators(); -} catch (ReplicationException e1) { - LOG.warn("Failure in getListOfReplicators(), will retry later", e1); - try { -Thread.sleep(ThreadLocalRandom.current().nextInt(1)); - } catch (InterruptedException e2) { -LOG.warn("Interrupted while waiting for list of replicators to be available, " -+ "will not adopt any abandoned queues", e2); -Thread.currentThread().interrupt(); -break; - } -} + try { +currentReplicators = replicationQueues.getListOfReplicators(); + } catch (ReplicationException e) { +server.abort("Failed to get all replicators", e); +return; } if (currentReplicators == null || currentReplicators.isEmpty()) { return;
[22/50] [abbrv] hbase git commit: HBASE-19642 Fix locking for peer modification procedure
HBASE-19642 Fix locking for peer modification procedure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/296d5868 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/296d5868 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/296d5868 Branch: refs/heads/HBASE-19397-branch-2 Commit: 296d58682c083417b87e91acdfcbb30cdb8b0593 Parents: 3b13faf Author: zhangduo Authored: Wed Dec 27 18:27:13 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../master/replication/ModifyPeerProcedure.java | 21 +--- 1 file changed, 18 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/296d5868/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java index 279fbc7..a682606 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java @@ -46,6 +46,8 @@ public abstract class ModifyPeerProcedure protected String peerId; + private volatile boolean locked; + // used to keep compatible with old client where we can only returns after updateStorage. protected ProcedurePrepareLatch latch; @@ -145,17 +147,30 @@ public abstract class ModifyPeerProcedure @Override protected LockState acquireLock(MasterProcedureEnv env) { -return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId) - ? LockState.LOCK_EVENT_WAIT - : LockState.LOCK_ACQUIRED; +if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) { + return LockState.LOCK_EVENT_WAIT; +} +locked = true; +return LockState.LOCK_ACQUIRED; } @Override protected void releaseLock(MasterProcedureEnv env) { +locked = false; env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId); } @Override + protected boolean holdLock(MasterProcedureEnv env) { +return true; + } + + @Override + protected boolean hasLock(MasterProcedureEnv env) { +return locked; + } + + @Override protected void rollbackState(MasterProcedureEnv env, PeerModificationState state) throws IOException, InterruptedException { if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
[42/50] [abbrv] hbase git commit: HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint
HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8cac522b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8cac522b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8cac522b Branch: refs/heads/HBASE-19397-branch-2 Commit: 8cac522bd78cfd3d7a866c552017ab71420cfb13 Parents: 24eff34 Author: zhangduo Authored: Fri Jan 5 18:28:44 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../RecoveredReplicationSource.java | 16 +- .../regionserver/ReplicationSource.java | 203 ++- 2 files changed, 116 insertions(+), 103 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8cac522b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java index 1be9a88..3cae0f2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java @@ -68,7 +68,7 @@ public class RecoveredReplicationSource extends ReplicationSource { LOG.debug("Someone has beat us to start a worker thread for wal group " + walGroupId); } else { LOG.debug("Starting up worker for wal group " + walGroupId); - worker.startup(getUncaughtExceptionHandler()); + worker.startup(this::uncaughtException); worker.setWALReader( startNewWALReader(worker.getName(), walGroupId, queue, worker.getStartPosition())); workerThreads.put(walGroupId, worker); @@ -76,13 +76,13 @@ public class RecoveredReplicationSource extends ReplicationSource { } @Override - protected ReplicationSourceWALReader startNewWALReader(String threadName, - String walGroupId, PriorityBlockingQueue queue, long startPosition) { -ReplicationSourceWALReader walReader = new RecoveredReplicationSourceWALReader(fs, -conf, queue, startPosition, walEntryFilter, this); -Threads.setDaemonThreadRunning(walReader, threadName -+ ".replicationSource.replicationWALReaderThread." + walGroupId + "," + queueId, - getUncaughtExceptionHandler()); + protected ReplicationSourceWALReader startNewWALReader(String threadName, String walGroupId, + PriorityBlockingQueue queue, long startPosition) { +ReplicationSourceWALReader walReader = + new RecoveredReplicationSourceWALReader(fs, conf, queue, startPosition, walEntryFilter, this); +Threads.setDaemonThreadRunning(walReader, + threadName + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + queueId, + this::uncaughtException); return walReader; } http://git-wip-us.apache.org/repos/asf/hbase/blob/8cac522b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 0092251..09b6cc1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -75,7 +75,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists; * */ @InterfaceAudience.Private -public class ReplicationSource extends Thread implements ReplicationSourceInterface { +public class ReplicationSource implements ReplicationSourceInterface { private static final Logger LOG = LoggerFactory.getLogger(ReplicationSource.class); // Queues of logs to process, entry in format of walGroupId->queue, @@ -114,10 +114,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf private MetricsSource metrics; // WARN threshold for the number of queued logs, defaults to 2 private int logQueueWarnThreshold; - // whether the replication endpoint has been initialized - private volatile boolean endpointInitialized = false; // ReplicationEndpoint which will handle the actual replication - private ReplicationEndpoint replicationEndpoint; + private volatile ReplicationEndpoint replicationEndpoint; // A filter (or a
[14/50] [abbrv] hbase git commit: HBASE-19520 Add UTs for the new lock type PEER
HBASE-19520 Add UTs for the new lock type PEER Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de68d00b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de68d00b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de68d00b Branch: refs/heads/HBASE-19397-branch-2 Commit: de68d00bdab986af405b9d646e270445d3674fd9 Parents: 50513f8 Author: Guanghao Zhang Authored: Wed Dec 20 16:43:38 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../procedure/MasterProcedureScheduler.java | 9 +- .../procedure/TestMasterProcedureScheduler.java | 65 - ...TestMasterProcedureSchedulerConcurrency.java | 135 +++ 3 files changed, 201 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/de68d00b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java index 58f9ccd..b18dd6c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java @@ -277,6 +277,13 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { while (tableIter.hasNext()) { count += tableIter.next().size(); } + +// Peer queues +final AvlTreeIterator peerIter = new AvlTreeIterator<>(peerMap); +while (peerIter.hasNext()) { + count += peerIter.next().size(); +} + return count; } @@ -807,7 +814,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { * @see #wakePeerExclusiveLock(Procedure, String) * @param procedure the procedure trying to acquire the lock * @param peerId peer to lock - * @return true if the procedure has to wait for the per to be available + * @return true if the procedure has to wait for the peer to be available */ public boolean waitPeerExclusiveLock(Procedure procedure, String peerId) { schedLock(); http://git-wip-us.apache.org/repos/asf/hbase/blob/de68d00b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java index 160f4d2..65757db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java @@ -908,6 +908,27 @@ public class TestMasterProcedureScheduler { } } + public static class TestPeerProcedure extends TestProcedure implements PeerProcedureInterface { +private final String peerId; +private final PeerOperationType opType; + +public TestPeerProcedure(long procId, String peerId, PeerOperationType opType) { + super(procId); + this.peerId = peerId; + this.opType = opType; +} + +@Override +public String getPeerId() { + return peerId; +} + +@Override +public PeerOperationType getPeerOperationType() { + return opType; +} + } + private static LockProcedure createLockProcedure(LockType lockType, long procId) throws Exception { LockProcedure procedure = new LockProcedure(); @@ -930,22 +951,19 @@ public class TestMasterProcedureScheduler { return createLockProcedure(LockType.SHARED, procId); } - private static void assertLockResource(LockedResource resource, - LockedResourceType resourceType, String resourceName) - { + private static void assertLockResource(LockedResource resource, LockedResourceType resourceType, + String resourceName) { assertEquals(resourceType, resource.getResourceType()); assertEquals(resourceName, resource.getResourceName()); } - private static void assertExclusiveLock(LockedResource resource, Procedure procedure) - { + private static void assertExclusiveLock(LockedResource resource, Procedure procedure) { assertEquals(LockType.EXCLUSIVE, resource.getLockType()); assertEquals(procedure, resource.getExclusiveLockOwnerProcedure()); assertEquals(0, resource.getSharedLockCount()); } - private static void assertSharedLock(LockedResource resource, int lockCount) -
[49/50] [abbrv] hbase git commit: HBASE-19873 addendum add missing rule for new tests
HBASE-19873 addendum add missing rule for new tests Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/05024640 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/05024640 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/05024640 Branch: refs/heads/HBASE-19397-branch-2 Commit: 050246402987ee5611e5af129589baf431d63bd5 Parents: c26916d Author: zhangduo Authored: Tue Jan 30 09:40:23 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../hbase/replication/TestZKReplicationPeerStorage.java | 7 ++- .../hbase/replication/TestZKReplicationQueueStorage.java | 8 +++- .../hbase/replication/TestReplicationProcedureRetry.java | 7 ++- .../apache/hadoop/hbase/util/TestHBaseFsckReplication.java | 6 ++ 4 files changed, 25 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java -- diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java index 3eb11da..3290fb0 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java @@ -33,19 +33,24 @@ import java.util.Map; import java.util.Random; import java.util.Set; import java.util.stream.Stream; - +import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @Category({ ReplicationTests.class, MediumTests.class }) public class TestZKReplicationPeerStorage { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestZKReplicationPeerStorage.class); + private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility(); private static ZKReplicationPeerStorage STORAGE; http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java -- diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java index 786730f..2c01a26 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java @@ -27,8 +27,8 @@ import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.SortedSet; - import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseZKTestingUtility; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -38,11 +38,17 @@ import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @Category({ ReplicationTests.class, MediumTests.class }) public class TestZKReplicationQueueStorage { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class); + private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility(); private static ZKReplicationQueueStorage STORAGE; http://git-wip-us.apache.org/repos/asf/hbase/blob/05024640/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java index ab35b46..a2ae0b4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
[30/50] [abbrv] hbase git commit: HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer
HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7091095b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7091095b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7091095b Branch: refs/heads/HBASE-19397-branch-2 Commit: 7091095bd45f2a0c0bb79acd69b5aaceb1e6 Parents: fef9964 Author: zhangduo Authored: Tue Jan 2 09:57:23 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:24 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 2 +- .../replication/VerifyReplication.java | 34 ++ .../hbase/replication/ReplicationPeers.java | 32 +++-- .../replication/ZKReplicationQueueStorage.java | 3 +- .../replication/ZKReplicationStorageBase.java | 4 +-- .../replication/TestReplicationStateBasic.java | 10 ++ .../master/replication/AddPeerProcedure.java| 5 ++- .../replication/DisablePeerProcedure.java | 3 +- .../master/replication/EnablePeerProcedure.java | 3 +- .../master/replication/ModifyPeerProcedure.java | 34 ++ .../replication/RefreshPeerProcedure.java | 17 - .../master/replication/RemovePeerProcedure.java | 7 ++-- .../replication/ReplicationPeerManager.java | 31 +++- .../replication/UpdatePeerConfigProcedure.java | 3 +- .../RemoteProcedureResultReporter.java | 3 +- .../regionserver/RefreshPeerCallable.java | 5 +-- .../regionserver/ReplicationSourceManager.java | 38 +++- .../TestReplicationAdminUsingProcedure.java | 7 ++-- 18 files changed, 124 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7091095b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index b80ee16..fdae288 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -27,8 +27,8 @@ import java.util.Set; import java.util.TreeMap; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; /** * A configuration for the replication peer cluster. http://git-wip-us.apache.org/repos/asf/hbase/blob/7091095b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index f0070f0..fe45762 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.mapreduce.replication; import java.io.IOException; import java.util.Arrays; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -45,13 +44,14 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -66,6 +66,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; imp
[26/50] [abbrv] hbase git commit: HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure
HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/febfe262 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/febfe262 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/febfe262 Branch: refs/heads/HBASE-19397-branch-2 Commit: febfe262446a999e785a6b2bed1a7cc7437848b0 Parents: 74f9e73 Author: zhangduo Authored: Mon Dec 18 15:22:36 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../procedure2/RemoteProcedureDispatcher.java | 3 +- .../src/main/protobuf/MasterProcedure.proto | 21 - .../src/main/protobuf/RegionServerStatus.proto | 3 +- .../src/main/protobuf/Replication.proto | 5 + .../replication/ReplicationPeersZKImpl.java | 4 +- .../org/apache/hadoop/hbase/master/HMaster.java | 95 --- .../hadoop/hbase/master/MasterRpcServices.java | 4 +- .../hadoop/hbase/master/MasterServices.java | 26 +++--- .../assignment/RegionTransitionProcedure.java | 13 +-- .../master/procedure/MasterProcedureEnv.java| 5 + .../master/procedure/ProcedurePrepareLatch.java | 2 +- .../master/replication/AddPeerProcedure.java| 97 .../replication/DisablePeerProcedure.java | 70 ++ .../master/replication/EnablePeerProcedure.java | 69 ++ .../master/replication/ModifyPeerProcedure.java | 97 +--- .../master/replication/RefreshPeerCallable.java | 67 -- .../replication/RefreshPeerProcedure.java | 28 -- .../master/replication/RemovePeerProcedure.java | 69 ++ .../master/replication/ReplicationManager.java | 76 --- .../replication/UpdatePeerConfigProcedure.java | 92 +++ .../hbase/regionserver/HRegionServer.java | 5 +- .../regionserver/RefreshPeerCallable.java | 70 ++ .../hbase/master/MockNoopMasterServices.java| 23 +++-- .../replication/DummyModifyPeerProcedure.java | 13 ++- 24 files changed, 733 insertions(+), 224 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java -- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java index 7e3dde6..fb852c3 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java @@ -247,9 +247,8 @@ public abstract class RemoteProcedureDispatcherhttp://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index d587ebf..e45e12f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -27,6 +27,7 @@ option optimize_for = SPEED; import "HBase.proto"; import "RPC.proto"; import "Snapshot.proto"; +import "Replication.proto"; // // WARNING - Compatibility rules @@ -368,9 +369,10 @@ message GCMergedRegionsStateData { } enum PeerModificationState { - UPDATE_PEER_STORAGE = 1; - REFRESH_PEER_ON_RS = 2; - POST_PEER_MODIFICATION = 3; + PRE_PEER_MODIFICATION = 1; + UPDATE_PEER_STORAGE = 2; + REFRESH_PEER_ON_RS = 3; + POST_PEER_MODIFICATION = 4; } message PeerModificationStateData { @@ -395,4 +397,17 @@ message RefreshPeerParameter { required string peer_id = 1; required PeerModificationType type = 2; required ServerName target_server = 3; +} + +message ModifyPeerStateData { + required string peer_id = 1; +} + +message AddPeerStateData { + required ReplicationPeer peer_config = 1; + required bool enabled = 2; +} + +message UpdatePeerConfigStateData { + required ReplicationPeer peer_config = 1; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/febfe262/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto index eb396ac..4f75941 100644 --- a/hbase-protocol-sh
[05/50] [abbrv] hbase git commit: HBASE-20061 HStore synchronized member variable filesCompacting should be private
HBASE-20061 HStore synchronized member variable filesCompacting should be private Signed-off-by: Chia-Ping Tsai Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/62b8da92 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/62b8da92 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/62b8da92 Branch: refs/heads/HBASE-19397-branch-2 Commit: 62b8da927e71af5489165b114481d0a62e58fba6 Parents: a66fa85 Author: Sean Busbey Authored: Fri Feb 23 14:38:03 2018 -0600 Committer: Sean Busbey Committed: Sat Feb 24 10:58:14 2018 -0600 -- .../main/java/org/apache/hadoop/hbase/regionserver/HStore.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/62b8da92/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index f20972e..220881d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -180,8 +180,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat private ScanInfo scanInfo; + // All access must be synchronized. // TODO: ideally, this should be part of storeFileManager, as we keep passing this to it. - final List filesCompacting = Lists.newArrayList(); + private final List filesCompacting = Lists.newArrayList(); // All access must be synchronized. private final Set changedReaderObservers =
[47/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/24eff344 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/24eff344 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/24eff344 Branch: refs/heads/HBASE-19397-branch-2 Commit: 24eff344ddc3541b7fa32401012a55861c432081 Parents: f42fc70 Author: Guanghao Zhang Authored: Thu Jan 4 16:58:01 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:58:44 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 1 - .../hbase/replication/ReplicationPeerImpl.java | 4 +- .../hbase/replication/ReplicationQueueInfo.java | 23 +- .../hbase/replication/ReplicationUtils.java | 56 ++ .../replication/TestReplicationStateZKImpl.java | 21 - .../regionserver/ReplicationSourceService.java | 3 +- .../regionserver/PeerProcedureHandler.java | 3 + .../regionserver/PeerProcedureHandlerImpl.java | 50 +- .../RecoveredReplicationSource.java | 6 +- .../RecoveredReplicationSourceShipper.java | 8 +- .../replication/regionserver/Replication.java | 11 +- .../regionserver/ReplicationSource.java | 34 +- .../regionserver/ReplicationSourceFactory.java | 4 +- .../ReplicationSourceInterface.java | 8 +- .../regionserver/ReplicationSourceManager.java | 827 ++- .../regionserver/ReplicationSourceShipper.java | 6 +- .../ReplicationSourceWALReader.java | 2 +- .../replication/ReplicationSourceDummy.java | 2 +- .../replication/TestNamespaceReplication.java | 57 +- .../TestReplicationSourceManager.java | 5 +- 20 files changed, 622 insertions(+), 509 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index fdae288..bf8d030 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; - import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java index 3e17025..604e0bb 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java @@ -1,5 +1,4 @@ -/* - * +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,6 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public class ReplicationPeerImpl implements ReplicationPeer { + private final Configuration conf; private final String id; http://git-wip-us.apache.org/repos/asf/hbase/blob/24eff344/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java index ecd888f..cd65f9b 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java @@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.ServerName; /** - * This class is responsible for the parsing logic for a znode representing a queue. + * This class is responsible for the parsing logic for a queue id representing a queue. * It will extract the peerId if it's recover
[17/50] [abbrv] hbase git commit: HBASE-19579 Add peer lock test for shell command list_locks
HBASE-19579 Add peer lock test for shell command list_locks Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59d57f82 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59d57f82 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59d57f82 Branch: refs/heads/HBASE-19397-branch-2 Commit: 59d57f82b40029700c996f2acf21196fa222ab78 Parents: b75cffe Author: Guanghao Zhang Authored: Sat Dec 23 21:04:27 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../src/main/protobuf/LockService.proto | 1 + .../src/test/ruby/shell/list_locks_test.rb | 19 +++ 2 files changed, 20 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/59d57f82/hbase-protocol-shaded/src/main/protobuf/LockService.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto b/hbase-protocol-shaded/src/main/protobuf/LockService.proto index b8d180c..0675070 100644 --- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto +++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto @@ -77,6 +77,7 @@ enum LockedResourceType { NAMESPACE = 2; TABLE = 3; REGION = 4; + PEER = 5; } message LockedResource { http://git-wip-us.apache.org/repos/asf/hbase/blob/59d57f82/hbase-shell/src/test/ruby/shell/list_locks_test.rb -- diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb b/hbase-shell/src/test/ruby/shell/list_locks_test.rb index f465a6b..ef1c0ce 100644 --- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb +++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb @@ -67,6 +67,25 @@ module Hbase proc_id) end +define_test 'list peer locks' do + lock = create_exclusive_lock(0) + peer_id = '1' + + @scheduler.waitPeerExclusiveLock(lock, peer_id) + output = capture_stdout { @list_locks.command } + @scheduler.wakePeerExclusiveLock(lock, peer_id) + + assert_equal( +"PEER(1)\n" \ +"Lock type: EXCLUSIVE, procedure: {" \ + "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ + "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ + "\"lastUpdate\"=>\"0\", " \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ +"}\n\n", +output) +end + define_test 'list server locks' do lock = create_exclusive_lock(0)
[16/50] [abbrv] hbase git commit: HBASE-19216 Implement a general framework to execute remote procedure on RS
HBASE-19216 Implement a general framework to execute remote procedure on RS Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/74f9e73a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/74f9e73a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/74f9e73a Branch: refs/heads/HBASE-19397-branch-2 Commit: 74f9e73a7a72a75e1aa96641e1b5fe0e0c2117c8 Parents: 309f336 Author: zhangduo Authored: Fri Dec 15 21:06:44 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../hbase/procedure2/LockedResourceType.java| 4 +- .../procedure2/RemoteProcedureDispatcher.java | 23 ++- .../src/main/protobuf/Admin.proto | 9 +- .../src/main/protobuf/MasterProcedure.proto | 30 +++ .../src/main/protobuf/RegionServerStatus.proto | 15 ++ .../apache/hadoop/hbase/executor/EventType.java | 26 ++- .../hadoop/hbase/executor/ExecutorType.java | 3 +- .../org/apache/hadoop/hbase/master/HMaster.java | 30 ++- .../hadoop/hbase/master/MasterRpcServices.java | 13 ++ .../assignment/RegionTransitionProcedure.java | 18 +- .../procedure/MasterProcedureScheduler.java | 113 ++- .../procedure/PeerProcedureInterface.java | 34 .../hbase/master/procedure/PeerQueue.java | 54 + .../master/procedure/RSProcedureDispatcher.java | 101 ++ .../hbase/master/procedure/SchemaLocking.java | 5 + .../master/replication/ModifyPeerProcedure.java | 127 .../master/replication/RefreshPeerCallable.java | 67 +++ .../replication/RefreshPeerProcedure.java | 197 +++ .../hbase/procedure2/RSProcedureCallable.java | 43 .../hbase/regionserver/HRegionServer.java | 61 ++ .../hbase/regionserver/RSRpcServices.java | 58 -- .../handler/RSProcedureHandler.java | 51 + .../assignment/TestAssignmentManager.java | 20 +- .../replication/DummyModifyPeerProcedure.java | 41 .../TestDummyModifyPeerProcedure.java | 80 .../security/access/TestAccessController.java | 1 + 26 files changed, 1109 insertions(+), 115 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/74f9e73a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java -- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java index c5fe62b..dc9b5d4 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,5 +22,5 @@ import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public enum LockedResourceType { - SERVER, NAMESPACE, TABLE, REGION + SERVER, NAMESPACE, TABLE, REGION, PEER } http://git-wip-us.apache.org/repos/asf/hbase/blob/74f9e73a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java -- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java index 6238e10..7e3dde6 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java @@ -226,13 +226,30 @@ public abstract class RemoteProcedureDispatcher - * @param */ public interface RemoteProcedure { +/** + * For building the remote operation. + */ RemoteOperation remoteCallBuild(TEnv env, TRemote remote); -void remoteCallCompleted(TEnv env, TRemote remote, RemoteOperation response); + +/** + * Called when the executeProcedure call is failed. + */ void remoteCallFailed(TEnv env, TRemote remote, IOException exception); + +/** + * Called when RS tells the remote procedure is succeeded through the + * {@code reportProcedureDone} method. + */ +void remoteOperationCompleted(TEnv env); + +/** + * Called when RS tells the remote procedure is failed through the {@code reportProcedureDone} + * method. + * @param error the error message + */ +void remoteOperationFailed(TEnv env, String error);
[11/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8f09f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java deleted file mode 100644 index b6f8784..000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java +++ /dev/null @@ -1,199 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.replication; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.ReplicationPeerNotFoundException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; -import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; -import org.apache.hadoop.hbase.replication.ReplicationPeers; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Manages and performs all replication admin operations. - * - * Used to add/remove a replication peer. - */ -@InterfaceAudience.Private -public class ReplicationManager { - private final ReplicationQueuesClient replicationQueuesClient; - private final ReplicationPeers replicationPeers; - - public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable abortable) - throws IOException { -try { - this.replicationQueuesClient = ReplicationFactory - .getReplicationQueuesClient(new ReplicationQueuesClientArguments(conf, abortable, zkw)); - this.replicationQueuesClient.init(); - this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, -this.replicationQueuesClient, abortable); - this.replicationPeers.init(); -} catch (Exception e) { - throw new IOException("Failed to construct ReplicationManager", e); -} - } - - public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled) - throws ReplicationException { -checkPeerConfig(peerConfig); -replicationPeers.registerPeer(peerId, peerConfig, enabled); -replicationPeers.peerConnected(peerId); - } - - public void removeReplicationPeer(String peerId) throws ReplicationException { -replicationPeers.peerDisconnected(peerId); -replicationPeers.unregisterPeer(peerId); - } - - public void enableReplicationPeer(String peerId) throws ReplicationException { -this.replicationPeers.enablePeer(peerId); - } - - public void disableReplicationPeer(String peerId) throws ReplicationException { -this.replicationPeers.disablePeer(peerId); - } - - public ReplicationPeerConfig getPeerConfig(String peerId) - throws ReplicationException, ReplicationPeerNotFoundException { -ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(peerId); -if (peerConfig == null) { - throw new ReplicationPeerNotFoundException(peerId); -} -return peerConfig; - } - - public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig) - throws ReplicationException, IOException { -checkPeerConfig(peerConfig); -this.replicationPeers.updatePeerConfig(peerId, peerConfig); - } - - public List listReplicationPeers(Pattern pattern) - throws ReplicationException { -List peers = new ArrayList<>(); -List peerIds = replicationPeers.getAllPeerIds(); -for (String peerId : peer
[07/50] [abbrv] hbase git commit: HBASE-20065 Revisit the timestamp usage in MetaTableAccessor
HBASE-20065 Revisit the timestamp usage in MetaTableAccessor Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2d5065bc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2d5065bc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2d5065bc Branch: refs/heads/HBASE-19397-branch-2 Commit: 2d5065bc7ef832ab84b11173e8b65d913de20bba Parents: 675f76b Author: zhangduo Authored: Sun Feb 25 09:13:34 2018 +0800 Committer: zhangduo Committed: Sun Feb 25 09:14:09 2018 +0800 -- .../apache/hadoop/hbase/MetaTableAccessor.java | 453 --- .../favored/FavoredNodeAssignmentHelper.java| 18 +- .../master/assignment/AssignmentManager.java| 4 - .../master/assignment/RegionStateStore.java | 54 +-- .../hadoop/hbase/regionserver/HRegion.java | 6 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 13 +- .../hadoop/hbase/util/HBaseFsckRepair.java | 4 +- .../hadoop/hbase/TestMetaTableAccessor.java | 54 +-- .../master/assignment/MockMasterServices.java | 8 - .../regionserver/TestGetClosestAtOrBefore.java | 109 ++--- 10 files changed, 301 insertions(+), 422 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2d5065bc/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 7d00f92..803f183 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -75,58 +75,65 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** - * Read/write operations on region and assignment information store in - * hbase:meta. + * + * Read/write operations on region and assignment information store in hbase:meta. + * + * + * Some of the methods of this class take ZooKeeperWatcher as a param. The only reason for this is + * because when used on client-side (like from HBaseAdmin), we want to use short-living connection + * (opened before each operation, closed right after), while when used on HM or HRS (like in + * AssignmentManager) we want permanent connection. + * + * + * HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the same table + * range (table, startKey, endKey). For every range, there will be at least one HRI defined which is + * called default replica. + * + * + * Meta layout (as of 0.98 + HBASE-10070) is like: * - * Some of the methods of this class take ZooKeeperWatcher as a param. The only reason - * for this is because when used on client-side (like from HBaseAdmin), we want to use - * short-living connection (opened before each operation, closed right after), while - * when used on HM or HRS (like in AssignmentManager) we want permanent connection. + * + * For each table there is single row in column family 'table' formatted: + *including namespace and columns are: + * table: state => contains table state + * + * For each table range, there is a single row, formatted like: + * , , , . + * This row corresponds to the regionName of the default region replica. + * Columns are: + * info:regioninfo => contains serialized HRI for the default region replica + * info:server => contains hostname:port (in string form) for the server hosting + *the default regionInfo replica + * info:server_ => contains hostname:port (in string form) for the server hosting + * the regionInfo replica with replicaId + * info:serverstartcode=> contains server start code (in binary long form) for the server + *hosting the default regionInfo replica + * info:serverstartcode_ => contains server start code (in binary long form) for + * the server hosting the regionInfo replica with + * replicaId + * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time + * the server opened the region with default replicaId + * info:seqnumDuringOpen_ => contains seqNum (in binary long form) for the region + * at the time the server opened the region with + * replicaId + * info:splitA => contains a serialized HRI for the first daughter region if the
[01/50] [abbrv] hbase git commit: HBASE-20048 Revert serial replication feature [Forced Update!]
Repository: hbase Updated Branches: refs/heads/HBASE-19397-branch-2 9cd0f7014 -> 732d22793 (forced update) http://git-wip-us.apache.org/repos/asf/hbase/blob/4ddfdaff/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java deleted file mode 100644 index 88513ea..000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java +++ /dev/null @@ -1,400 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.HTestConst; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.ReplicationTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Category({ ReplicationTests.class, LargeTests.class }) -public class TestSerialReplication { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSerialReplication.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestSerialReplication.class); - - private static Configuration conf1; - private static Configuration conf2; - - private static HBaseTestingUtility utility1; - private static HBaseTestingUtility utility2; - - private static final byte[] famName = Bytes.toBytes("f"); - private static final byte[] VALUE = Bytes.toBytes("v"); - private static final byte[] ROW = Bytes.toBytes("r"); - private static final byte[][] ROWS = HTestConst.makeNAscii(ROW, 100); - - @Rule - public TestName name = new TestName(); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { -conf1 = HBaseConfiguration.create(); -conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); -// smaller block size and capacity to trigger more operations -// and test them -conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20); -conf1.setInt("replication.source.size.capacity", 1024); -conf1.setLong("replication.source.sleepforretries", 100); -conf1.setInt("hbase.regionserver.maxlogs", 10); -conf1.setLong("hbase.master.logcleaner.ttl", 10); -conf1.setBoolean("dfs.support.append", true); -conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); -conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KE
[33/50] [abbrv] hbase git commit: HBASE-19623 Create replication endpoint asynchronously when adding a replication source
HBASE-19623 Create replication endpoint asynchronously when adding a replication source Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a762b60 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a762b60 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a762b60 Branch: refs/heads/HBASE-19397-branch-2 Commit: 8a762b6047f5b9483c955544a4f8f139e5982e60 Parents: 7091095 Author: zhangduo Authored: Tue Jan 2 13:25:58 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:58 2018 +0800 -- .../hbase/replication/ReplicationPeer.java | 8 ++ .../hbase/replication/ReplicationPeers.java | 18 +-- .../replication/ZKReplicationPeerStorage.java | 7 +- .../replication/TestReplicationStateBasic.java | 20 +--- .../TestZKReplicationPeerStorage.java | 14 +-- .../HBaseInterClusterReplicationEndpoint.java | 17 ++- .../RecoveredReplicationSource.java | 13 +-- .../regionserver/ReplicationSource.java | 110 +++ .../ReplicationSourceInterface.java | 8 +- .../regionserver/ReplicationSourceManager.java | 47 +--- .../client/TestAsyncReplicationAdminApi.java| 2 - .../replication/TestReplicationAdmin.java | 2 - .../replication/ReplicationSourceDummy.java | 7 +- .../replication/TestReplicationSource.java | 5 +- .../TestReplicationSourceManager.java | 8 +- 15 files changed, 116 insertions(+), 170 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java index 4846018..2da3cce 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java @@ -54,6 +54,14 @@ public interface ReplicationPeer { PeerState getPeerState(); /** + * Test whether the peer is enabled. + * @return {@code true} if enabled, otherwise {@code false}. + */ + default boolean isPeerEnabled() { +return getPeerState() == PeerState.ENABLED; + } + + /** * Get the peer config object * @return the ReplicationPeerConfig for this peer */ http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java index 422801b..45940a5 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.replication; import java.io.IOException; +import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -86,21 +87,6 @@ public class ReplicationPeers { } /** - * Get the peer state for the specified connected remote slave cluster. The value might be read - * from cache, so it is recommended to use {@link #peerStorage } to read storage directly if - * reading the state after enabling or disabling it. - * @param peerId a short that identifies the cluster - * @return true if replication is enabled, false otherwise. - */ - public boolean isPeerEnabled(String peerId) { -ReplicationPeer replicationPeer = this.peerCache.get(peerId); -if (replicationPeer == null) { - throw new IllegalArgumentException("Peer with id= " + peerId + " is not cached"); -} -return replicationPeer.getPeerState() == PeerState.ENABLED; - } - - /** * Returns the ReplicationPeerImpl for the specified cached peer. This ReplicationPeer will * continue to track changes to the Peer's state and config. This method returns null if no peer * has been cached with the given peerId. @@ -117,7 +103,7 @@ public class ReplicationPeers { * @return a Set of Strings for peerIds */ public Set getAllPeerIds() { -return peerCache.keySet(); +return Collections.unmodifiableSet(peerCache.keySet()); } public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig, http://git-wip-us.apache.org/repos/asf/hbase/blob/8a762b60/hbase
[35/50] [abbrv] hbase git commit: HBASE-19544 Add UTs for testing concurrent modifications on replication peer
HBASE-19544 Add UTs for testing concurrent modifications on replication peer Signed-off-by: zhangduo Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1d7960ca Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1d7960ca Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1d7960ca Branch: refs/heads/HBASE-19397-branch-2 Commit: 1d7960cada9f9e0b3a1319ade155c0f4f5d3b8c9 Parents: a75ee53 Author: Guanghao Zhang Authored: Tue Jan 2 17:07:41 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 15:56:58 2018 +0800 -- .../replication/TestReplicationAdmin.java | 69 1 file changed, 69 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/1d7960ca/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java index 772a9d6..a753d23 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -58,6 +59,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Unit testing of ReplicationAdmin @@ -69,6 +72,8 @@ public class TestReplicationAdmin { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestReplicationAdmin.class); + private static final Logger LOG = LoggerFactory.getLogger(TestReplicationAdmin.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -118,6 +123,70 @@ public class TestReplicationAdmin { } @Test + public void testConcurrentPeerOperations() throws Exception { +int threadNum = 5; +AtomicLong successCount = new AtomicLong(0); + +// Test concurrent add peer operation +Thread[] addPeers = new Thread[threadNum]; +for (int i = 0; i < threadNum; i++) { + addPeers[i] = new Thread(() -> { +try { + hbaseAdmin.addReplicationPeer(ID_ONE, +ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build()); + successCount.incrementAndGet(); +} catch (Exception e) { + LOG.debug("Got exception when add replication peer", e); +} + }); + addPeers[i].start(); +} +for (Thread addPeer : addPeers) { + addPeer.join(); +} +assertEquals(1, successCount.get()); + +// Test concurrent remove peer operation +successCount.set(0); +Thread[] removePeers = new Thread[threadNum]; +for (int i = 0; i < threadNum; i++) { + removePeers[i] = new Thread(() -> { +try { + hbaseAdmin.removeReplicationPeer(ID_ONE); + successCount.incrementAndGet(); +} catch (Exception e) { + LOG.debug("Got exception when remove replication peer", e); +} + }); + removePeers[i].start(); +} +for (Thread removePeer : removePeers) { + removePeer.join(); +} +assertEquals(1, successCount.get()); + +// Test concurrent add peer operation again +successCount.set(0); +addPeers = new Thread[threadNum]; +for (int i = 0; i < threadNum; i++) { + addPeers[i] = new Thread(() -> { +try { + hbaseAdmin.addReplicationPeer(ID_ONE, +ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build()); + successCount.incrementAndGet(); +} catch (Exception e) { + LOG.debug("Got exception when add replication peer", e); +} + }); + addPeers[i].start(); +} +for (Thread addPeer : addPeers) { + addPeer.join(); +} +assertEquals(1, successCount.get()); + } + + @Test public void testAddInvalidPeer() { ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder(); builder.setClusterKey(KEY_ONE);
[10/50] [abbrv] hbase git commit: HBASE-20019 Document the ColumnValueFilter
HBASE-20019 Document the ColumnValueFilter Signed-off-by: Chia-Ping Tsai Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/309f3360 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/309f3360 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/309f3360 Branch: refs/heads/HBASE-19397-branch-2 Commit: 309f3360bf3f45c7ea572a09eddada02cf6b93b3 Parents: 10270e3 Author: Reid Chan Authored: Mon Feb 26 11:31:08 2018 +0800 Committer: Chia-Ping Tsai Committed: Mon Feb 26 15:11:01 2018 +0800 -- src/main/asciidoc/_chapters/architecture.adoc | 35 ++ 1 file changed, 35 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/309f3360/src/main/asciidoc/_chapters/architecture.adoc -- diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc index 5ba81c1..b29244c 100644 --- a/src/main/asciidoc/_chapters/architecture.adoc +++ b/src/main/asciidoc/_chapters/architecture.adoc @@ -321,6 +321,41 @@ SingleColumnValueFilter filter = new SingleColumnValueFilter( scan.setFilter(filter); +[[client.filter.cv.cvf]] + ColumnValueFilter + +Introduced in HBase-2.0.0 version as a complementation of SingleColumnValueFilter, ColumnValueFilter +gets matched cell only, while SingleColumnValueFilter gets the entire row +(has other columns and values) to which the matched cell belongs. Parameters of constructor of +ColumnValueFilter are the same as SingleColumnValueFilter. +[source,java] + +ColumnValueFilter filter = new ColumnValueFilter( + cf, + column, + CompareOperaor.EQUAL, + Bytes.toBytes("my value") + ); +scan.setFilter(filter); + + +Note. For simple query like "equals to a family:qualifier:value", we highly recommend to use the +following way instead of using SingleColumnValueFilter or ColumnValueFilter: +[source,java] + +Scan scan = new Scan(); +scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qualifier")); +ValueFilter vf = new ValueFilter(CompareOperator.EQUAL, + new BinaryComparator(Bytes.toBytes("value"))); +scan.setFilter(vf); +... + +This scan will restrict to the specified column 'family:qualifier', avoiding scan unrelated +families and columns, which has better performance, and `ValueFilter` is the condition used to do +the value filtering. + +But if query is much more complicated beyond this book, then please make your good choice case by case. + [[client.filter.cvp]] === Column Value Comparators
[31/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bbc6071/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index b8d80d2..5b7bab8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -33,17 +33,20 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -56,7 +59,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.ReplicationQueueInfo; -import org.apache.hadoop.hbase.replication.ReplicationQueues; +import org.apache.hadoop.hbase.replication.ReplicationQueueStorage; import org.apache.hadoop.hbase.replication.ReplicationTracker; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; @@ -91,7 +94,7 @@ public class ReplicationSourceManager implements ReplicationListener { private final List sources; // List of all the sources we got from died RSs private final List oldsources; - private final ReplicationQueues replicationQueues; + private final ReplicationQueueStorage queueStorage; private final ReplicationTracker replicationTracker; private final ReplicationPeers replicationPeers; // UUID for this cluster @@ -124,7 +127,7 @@ public class ReplicationSourceManager implements ReplicationListener { /** * Creates a replication manager and sets the watch on all the other registered region servers - * @param replicationQueues the interface for manipulating replication queues + * @param queueStorage the interface for manipulating replication queues * @param replicationPeers * @param replicationTracker * @param conf the configuration to use @@ -134,14 +137,14 @@ public class ReplicationSourceManager implements ReplicationListener { * @param oldLogDir the directory where old logs are archived * @param clusterId */ - public ReplicationSourceManager(ReplicationQueues replicationQueues, + public ReplicationSourceManager(ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, ReplicationTracker replicationTracker, Configuration conf, Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID clusterId, WALFileLengthProvider walFileLengthProvider) throws IOException { //CopyOnWriteArrayList is thread-safe. //Generally, reading is more than modifying. this.sources = new CopyOnWriteArrayList<>(); -this.replicationQueues = replicationQueues; +this.queueStorage = queueStorage; this.replicationPeers = replicationPeers; this.replicationTracker = replicationTracker; this.server = server; @@ -174,6 +177,19 @@ public class ReplicationSourceManager implements ReplicationListener { HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT); } + @FunctionalInterface + private interface ReplicationQueueOperation { +void exec() throws ReplicationException; + } + + private void abortWhenFail(ReplicationQueueOperation op) { +try { + op.exec(); +} catch (ReplicationException e) { + server.abort("Failed to operate on replication queue", e); +} + } + /** * Provide the id of the peer and a log key and this method will figure which * wal it belongs to and will log, for this region server, the current @@ -185,12 +201,13 @@ public class ReplicationSourceManager implements ReplicationListener { * @param queueRecovered indicates if this queue comes from another region server * @param holdLogInZK if true then the log is retained in
[21/50] [abbrv] hbase git commit: HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/751a6eb8 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/751a6eb8 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/751a6eb8 Branch: refs/heads/HBASE-19397-branch-2 Commit: 751a6eb8ff88ec1ccc4df027181a8c01323c640c Parents: 59d57f8 Author: Guanghao Zhang Authored: Tue Dec 26 11:39:34 2017 +0800 Committer: zhangduo Committed: Mon Feb 26 15:52:10 2018 +0800 -- .../replication/VerifyReplication.java | 5 - .../hbase/replication/ReplicationPeer.java | 42 ++-- .../hbase/replication/ReplicationPeerImpl.java | 169 ++ .../replication/ReplicationPeerZKImpl.java | 233 --- .../hbase/replication/ReplicationPeers.java | 4 +- .../replication/ReplicationPeersZKImpl.java | 23 +- .../replication/TestReplicationStateBasic.java | 7 +- .../regionserver/PeerProcedureHandlerImpl.java | 29 +-- 8 files changed, 216 insertions(+), 296 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/751a6eb8/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index 9065f4e..09d4b4b 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl; import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; @@ -333,7 +332,6 @@ public class VerifyReplication extends Configured implements Tool { private static Pair getPeerQuorumConfig( final Configuration conf, String peerId) throws IOException { ZKWatcher localZKW = null; -ReplicationPeerZKImpl peer = null; try { localZKW = new ZKWatcher(conf, "VerifyReplication", new Abortable() { @@ -354,9 +352,6 @@ public class VerifyReplication extends Configured implements Tool { throw new IOException( "An error occurred while trying to connect to the remove peer cluster", e); } finally { - if (peer != null) { -peer.close(); - } if (localZKW != null) { localZKW.close(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/751a6eb8/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java -- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java index b66d76d..4846018 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; - /** * ReplicationPeer manages enabled / disabled state for the peer. */ @@ -49,65 +48,52 @@ public interface ReplicationPeer { String getId(); /** - * Get the peer config object - * @return the ReplicationPeerConfig for this peer - */ - public ReplicationPeerConfig getPeerConfig(); - - /** - * Get the peer config object. if loadFromBackingStore is true, it will load from backing store - * directly and update its load peer config. otherwise, just return the local cached peer config. - * @return the ReplicationPeerConfig for this peer - */ - public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore) - throws ReplicationException; - - /** * Returns the state of the peer by reading local cache. * @return the enabled state */ PeerState getPeerState(); /** - * Returns the state of peer, if loadFromBackingStore is true, it will load from backing store - * directly and update its local peer state. otherwise, just return t
[18/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75cffe0/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java index 6e27a21..d8f9625 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java @@ -21,13 +21,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; @@ -48,17 +48,18 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.ReplicationQueueInfo; +import org.apache.hadoop.hbase.replication.ReplicationQueueStorage; import org.apache.hadoop.hbase.replication.ReplicationQueues; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments; +import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; +import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; import org.apache.hadoop.hbase.replication.ReplicationTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap; /** @@ -303,57 +304,53 @@ public class DumpReplicationQueues extends Configured implements Tool { } public String dumpQueues(ClusterConnection connection, ZKWatcher zkw, Set peerIds, - boolean hdfs) throws Exception { -ReplicationQueuesClient queuesClient; + boolean hdfs) throws Exception { +ReplicationQueueStorage queueStorage; ReplicationPeers replicationPeers; ReplicationQueues replicationQueues; ReplicationTracker replicationTracker; -ReplicationQueuesClientArguments replicationArgs = -new ReplicationQueuesClientArguments(getConf(), new WarnOnlyAbortable(), zkw); +ReplicationQueuesArguments replicationArgs = +new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(), zkw); StringBuilder sb = new StringBuilder(); -queuesClient = ReplicationFactory.getReplicationQueuesClient(replicationArgs); -queuesClient.init(); +queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf()); replicationQueues = ReplicationFactory.getReplicationQueues(replicationArgs); -replicationPeers = ReplicationFactory.getReplicationPeers(zkw, getConf(), queuesClient, connection); +replicationPeers = +ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage, connection); replicationTracker = ReplicationFactory.getReplicationTracker(zkw, replicationPeers, getConf(), new WarnOnlyAbortable(), new WarnOnlyStoppable()); -List liveRegionServers = replicationTracker.getListOfRegionServers(); +Set liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers()); // Loops each peer on each RS and dumps the queues -try { - List regionservers = queuesClient.getListOfReplicators(); - if (regionservers == null || regionservers.isEmpty()) { -return sb.toString(); +List regionservers = queueStorage.getListOfReplicators(); +if (regionservers == null || regionservers.isEmpty()) { + return sb.toString(); +} +for (ServerName regionserver : regionservers) { + List queueIds = queueStorage.getAllQueues(regionserver); + replicationQueues.init(regionserver.getServerName()); + if (!liveRegionServers.contains(regionserver.getServerName())) { +deadRegionServers.add(regionserver.getServerName()); } - for (String regionserver : regionservers) { -List queueIds = queuesClient.getAllQueues(regionserver); -replicationQueues.init(regionserver); -if (!liveRegionServers.contains(regionserver)) { - deadRegionServers.add(regionserver); -} -for (String
hbase git commit: HBASE-20083 Fix findbugs error for ReplicationSyncUp
Repository: hbase Updated Branches: refs/heads/master 8c74d177f -> 2beda62a1 HBASE-20083 Fix findbugs error for ReplicationSyncUp Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2beda62a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2beda62a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2beda62a Branch: refs/heads/master Commit: 2beda62a10f0828eb10cec28b0ba53246cd0b671 Parents: 8c74d17 Author: zhangduo Authored: Mon Feb 26 16:37:58 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 22:13:13 2018 +0800 -- .../regionserver/ReplicationSyncUp.java | 59 ++-- .../replication/TestReplicationSyncUpTool.java | 6 +- 2 files changed, 19 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2beda62a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java index 283eb96..c2862de 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java @@ -49,35 +49,18 @@ import org.apache.hadoop.util.ToolRunner; */ public class ReplicationSyncUp extends Configured implements Tool { - private static Configuration conf; - private static final long SLEEP_TIME = 1; - // although the tool is designed to be run on command line - // this api is provided for executing the tool through another app - public static void setConfigure(Configuration config) { -conf = config; - } - /** * Main program - * @param args - * @throws Exception */ public static void main(String[] args) throws Exception { -if (conf == null) conf = HBaseConfiguration.create(); -int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args); +int ret = ToolRunner.run(HBaseConfiguration.create(), new ReplicationSyncUp(), args); System.exit(ret); } @Override public int run(String[] args) throws Exception { -Replication replication; -ReplicationSourceManager manager; -FileSystem fs; -Path oldLogDir, logDir, walRootDir; -ZKWatcher zkw; - Abortable abortable = new Abortable() { @Override public void abort(String why, Throwable e) { @@ -88,23 +71,19 @@ public class ReplicationSyncUp extends Configured implements Tool { return false; } }; - -zkw = -new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(), abortable, -true); - -walRootDir = FSUtils.getWALRootDir(conf); -fs = FSUtils.getWALFileSystem(conf); -oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); -logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); - -System.out.println("Start Replication Server start"); -replication = new Replication(); -replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null); -manager = replication.getReplicationManager(); -manager.init().get(); - -try { +Configuration conf = getConf(); +try (ZKWatcher zkw = + new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(), abortable, true)) { + Path walRootDir = FSUtils.getWALRootDir(conf); + FileSystem fs = FSUtils.getWALFileSystem(conf); + Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); + + System.out.println("Start Replication Server start"); + Replication replication = new Replication(); + replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null); + ReplicationSourceManager manager = replication.getReplicationManager(); + manager.init().get(); while (manager.activeFailoverTaskCount() > 0) { Thread.sleep(SLEEP_TIME); } @@ -114,15 +93,12 @@ public class ReplicationSyncUp extends Configured implements Tool { manager.join(); } catch (InterruptedException e) { System.err.println("didn't wait long enough:" + e); - return (-1); -} finally { - zkw.close(); + return -1; } - return 0; } - static class DummyServer implements Server { + class DummyServer implements Server { String hostname; ZKWatcher zkw; @@ -138,7 +114,7 @@ public class ReplicationSyncUp extends Configured implements Tool { @Override public Configuration getConfiguration() { - return conf;
hbase git commit: HBASE-20083 Fix findbugs error for ReplicationSyncUp
Repository: hbase Updated Branches: refs/heads/branch-2 309f3360b -> ef02762dd HBASE-20083 Fix findbugs error for ReplicationSyncUp Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ef02762d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ef02762d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ef02762d Branch: refs/heads/branch-2 Commit: ef02762dd8ac314c4d027deaf9cc5a913d68b791 Parents: 309f336 Author: zhangduo Authored: Mon Feb 26 16:37:58 2018 +0800 Committer: zhangduo Committed: Mon Feb 26 22:16:37 2018 +0800 -- .../regionserver/ReplicationSyncUp.java | 79 +++- .../replication/TestReplicationSyncUpTool.java | 6 +- 2 files changed, 27 insertions(+), 58 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/ef02762d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java index 43c7bce..f7ea78f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.replication.regionserver; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -37,51 +36,31 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** - * In a scenario of Replication based Disaster/Recovery, when hbase - * Master-Cluster crashes, this tool is used to sync-up the delta from Master to - * Slave using the info from ZooKeeper. The tool will run on Master-Cluser, and - * assume ZK, Filesystem and NetWork still available after hbase crashes - * + * In a scenario of Replication based Disaster/Recovery, when hbase Master-Cluster crashes, this + * tool is used to sync-up the delta from Master to Slave using the info from ZooKeeper. The tool + * will run on Master-Cluser, and assume ZK, Filesystem and NetWork still available after hbase + * crashes + * + * * hbase org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp + * */ - public class ReplicationSyncUp extends Configured implements Tool { - private static final Logger LOG = LoggerFactory.getLogger(ReplicationSyncUp.class.getName()); - - private static Configuration conf; - private static final long SLEEP_TIME = 1; - // although the tool is designed to be run on command line - // this api is provided for executing the tool through another app - public static void setConfigure(Configuration config) { -conf = config; - } - /** * Main program - * @param args - * @throws Exception */ public static void main(String[] args) throws Exception { -if (conf == null) conf = HBaseConfiguration.create(); -int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args); +int ret = ToolRunner.run(HBaseConfiguration.create(), new ReplicationSyncUp(), args); System.exit(ret); } @Override public int run(String[] args) throws Exception { -Replication replication; -ReplicationSourceManager manager; -FileSystem fs; -Path oldLogDir, logDir, walRootDir; -ZKWatcher zkw; - Abortable abortable = new Abortable() { @Override public void abort(String why, Throwable e) { @@ -92,23 +71,19 @@ public class ReplicationSyncUp extends Configured implements Tool { return false; } }; - -zkw = -new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(), abortable, -true); - -walRootDir = FSUtils.getWALRootDir(conf); -fs = FSUtils.getWALFileSystem(conf); -oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); -logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); - -System.out.println("Start Replication Server start"); -replication = new Replication(); -replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null); -manager = replication.getReplicationManager(); -manager.init(); - -try { +Configuration conf = getConf(); +try (ZKWatcher zkw = + new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(), abortable, true)) { + Path walRootDir =
hbase git commit: HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods
Repository: hbase Updated Branches: refs/heads/master 2beda62a1 -> a29b3caf4 HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a29b3caf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a29b3caf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a29b3caf Branch: refs/heads/master Commit: a29b3caf4dbc7b8833474ef5da5438f7f6907e00 Parents: 2beda62 Author: haxiaolin Authored: Mon Feb 26 14:25:01 2018 +0800 Committer: tedyu Committed: Mon Feb 26 07:30:56 2018 -0800 -- .../hbase/rsgroup/RSGroupAdminServer.java | 8 +++- .../hadoop/hbase/rsgroup/TestRSGroups.java | 7 ++- .../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 46 3 files changed, 40 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java -- diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index aba57fe..094fc1d 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -675,8 +675,12 @@ public class RSGroupAdminServer implements RSGroupAdmin { private void checkForDeadOrOnlineServers(Set servers) throws ConstraintException { // This uglyness is because we only have Address, not ServerName. Set onlineServers = new HashSet<>(); -for(ServerName server: master.getServerManager().getOnlineServers().keySet()) { - onlineServers.add(server.getAddress()); +List drainingServers = master.getServerManager().getDrainingServersList(); +for (ServerName server : master.getServerManager().getOnlineServers().keySet()) { + // Only online but not decommissioned servers are really online + if (!drainingServers.contains(server)) { +onlineServers.add(server.getAddress()); + } } Set deadServers = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java -- diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java index 9116f3b..610278a 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.net.Address; @@ -66,7 +65,6 @@ public class TestRSGroups extends TestRSGroupsBase { HBaseClassTestRule.forClass(TestRSGroups.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroups.class); - private static HMaster master; private static boolean INIT = false; private static RSGroupAdminEndpoint rsGroupAdminEndpoint; @@ -126,6 +124,11 @@ public class TestRSGroups extends TestRSGroupsBase { deleteNamespaceIfNecessary(); deleteGroups(); +for(ServerName sn : admin.listDecommissionedRegionServers()){ + admin.recommissionRegionServer(sn, null); +} +assertTrue(admin.listDecommissionedRegionServers().isEmpty()); + int missing = NUM_SLAVES_BASE - getNumServers(); LOG.info("Restoring servers: "+missing); for(int i=0; ihttp://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java -- diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index c3f7eef..76bcd20 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -25,8 +25,10 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.secur
hbase git commit: HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods
Repository: hbase Updated Branches: refs/heads/branch-2 ef02762dd -> 0bf33c802 HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0bf33c80 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0bf33c80 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0bf33c80 Branch: refs/heads/branch-2 Commit: 0bf33c802d7ad194f39cb80e86a0255b2155f455 Parents: ef02762 Author: haxiaolin Authored: Mon Feb 26 14:25:01 2018 +0800 Committer: tedyu Committed: Mon Feb 26 07:31:58 2018 -0800 -- .../hbase/rsgroup/RSGroupAdminServer.java | 8 +++- .../hadoop/hbase/rsgroup/TestRSGroups.java | 7 ++- .../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 46 3 files changed, 40 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java -- diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 57d5412..f7e7731 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -675,8 +675,12 @@ public class RSGroupAdminServer implements RSGroupAdmin { private void checkForDeadOrOnlineServers(Set servers) throws ConstraintException { // This uglyness is because we only have Address, not ServerName. Set onlineServers = new HashSet<>(); -for(ServerName server: master.getServerManager().getOnlineServers().keySet()) { - onlineServers.add(server.getAddress()); +List drainingServers = master.getServerManager().getDrainingServersList(); +for (ServerName server : master.getServerManager().getOnlineServers().keySet()) { + // Only online but not decommissioned servers are really online + if (!drainingServers.contains(server)) { +onlineServers.add(server.getAddress()); + } } Set deadServers = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java -- diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java index 9116f3b..610278a 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.net.Address; @@ -66,7 +65,6 @@ public class TestRSGroups extends TestRSGroupsBase { HBaseClassTestRule.forClass(TestRSGroups.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroups.class); - private static HMaster master; private static boolean INIT = false; private static RSGroupAdminEndpoint rsGroupAdminEndpoint; @@ -126,6 +124,11 @@ public class TestRSGroups extends TestRSGroupsBase { deleteNamespaceIfNecessary(); deleteGroups(); +for(ServerName sn : admin.listDecommissionedRegionServers()){ + admin.recommissionRegionServer(sn, null); +} +assertTrue(admin.listDecommissionedRegionServers().isEmpty()); + int missing = NUM_SLAVES_BASE - getNumServers(); LOG.info("Restoring servers: "+missing); for(int i=0; ihttp://git-wip-us.apache.org/repos/asf/hbase/blob/0bf33c80/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java -- diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index cae9da3..fdc29b3 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -25,8 +25,10 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.s
hbase git commit: HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Repository: hbase Updated Branches: refs/heads/branch-2 0bf33c802 -> 372c68fca HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/372c68fc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/372c68fc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/372c68fc Branch: refs/heads/branch-2 Commit: 372c68fca28383d08128ce1c376492706b1d66f3 Parents: 0bf33c8 Author: Vasudevan Authored: Mon Feb 26 22:05:12 2018 +0530 Committer: Vasudevan Committed: Mon Feb 26 22:06:28 2018 +0530 -- ...estAvoidCellReferencesIntoShippedBlocks.java | 30 +++- 1 file changed, 17 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/372c68fc/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java index 0e12ad6..d22772a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java @@ -400,23 +400,27 @@ public class TestAvoidCellReferencesIntoShippedBlocks { scanner = table.getScanner(s1); int count = Iterables.size(scanner); assertEquals("Count the rows", 2, count); - iterator = cache.iterator(); - List newCacheList = new ArrayList<>(); - while (iterator.hasNext()) { -CachedBlock next = iterator.next(); -BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset()); -newCacheList.add(cacheKey); - } int newBlockRefCount = 0; - for (BlockCacheKey key : cacheList) { -if (newCacheList.contains(key)) { - newBlockRefCount++; + List newCacheList = new ArrayList<>(); + while (true) { +newBlockRefCount = 0; +newCacheList.clear(); +iterator = cache.iterator(); +while (iterator.hasNext()) { + CachedBlock next = iterator.next(); + BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset()); + newCacheList.add(cacheKey); +} +for (BlockCacheKey key : cacheList) { + if (newCacheList.contains(key)) { +newBlockRefCount++; + } +} +if (newBlockRefCount == 6) { + break; } } - - assertEquals("old blocks should still be found ", 6, newBlockRefCount); latch.countDown(); - } catch (IOException e) { } }
hbase git commit: HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Repository: hbase Updated Branches: refs/heads/master a29b3caf4 -> 7cfb46432 HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cfb4643 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cfb4643 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cfb4643 Branch: refs/heads/master Commit: 7cfb46432fbdf9b53592be11efc8a7d79d1a9455 Parents: a29b3ca Author: Vasudevan Authored: Mon Feb 26 22:07:42 2018 +0530 Committer: Vasudevan Committed: Mon Feb 26 22:07:42 2018 +0530 -- ...estAvoidCellReferencesIntoShippedBlocks.java | 30 +++- 1 file changed, 17 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/7cfb4643/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java index 0e12ad6..d22772a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java @@ -400,23 +400,27 @@ public class TestAvoidCellReferencesIntoShippedBlocks { scanner = table.getScanner(s1); int count = Iterables.size(scanner); assertEquals("Count the rows", 2, count); - iterator = cache.iterator(); - List newCacheList = new ArrayList<>(); - while (iterator.hasNext()) { -CachedBlock next = iterator.next(); -BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset()); -newCacheList.add(cacheKey); - } int newBlockRefCount = 0; - for (BlockCacheKey key : cacheList) { -if (newCacheList.contains(key)) { - newBlockRefCount++; + List newCacheList = new ArrayList<>(); + while (true) { +newBlockRefCount = 0; +newCacheList.clear(); +iterator = cache.iterator(); +while (iterator.hasNext()) { + CachedBlock next = iterator.next(); + BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset()); + newCacheList.add(cacheKey); +} +for (BlockCacheKey key : cacheList) { + if (newCacheList.contains(key)) { +newBlockRefCount++; + } +} +if (newBlockRefCount == 6) { + break; } } - - assertEquals("old blocks should still be found ", 6, newBlockRefCount); latch.countDown(); - } catch (IOException e) { } }
[6/6] hbase git commit: HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky (cherry picked from commit 08b993997404aad7ec8f66b8c2b2083062c9ef1a) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41cd37cf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41cd37cf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41cd37cf Branch: refs/heads/branch-1.2 Commit: 41cd37cffc690d056edf1ca10d5cbc07d02373c5 Parents: 4b79555 Author: Chia-Ping Tsai Authored: Sun Feb 18 21:22:58 2018 +0800 Committer: Sean Busbey Committed: Mon Feb 26 10:46:57 2018 -0600 -- .../TestCatalogJanitorInMemoryStates.java | 52 +--- 1 file changed, 22 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/41cd37cf/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java index 6ea7e2f..40396ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java @@ -124,7 +124,7 @@ public class TestCatalogJanitorInMemoryStates { * AM's serverHoldings and */ @Test(timeout = 18) - public void testInMemoryForReplicaParentCleanup() throws IOException, InterruptedException { + public void testInMemoryForReplicaParentCleanup() throws Exception { final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); final CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().catalogJanitorChore; @@ -139,35 +139,27 @@ public class TestCatalogJanitorInMemoryStates { // There are two regions, one for primary, one for the replica. assertTrue(allRegionLocations.size() == 2); -HRegionLocation replicaParentRegion, primaryParentRegion; -if (RegionReplicaUtil.isDefaultReplica( -allRegionLocations.get(0).getRegionInfo().getReplicaId())) { - primaryParentRegion = allRegionLocations.get(0); - replicaParentRegion = allRegionLocations.get(1); -} else { - primaryParentRegion = allRegionLocations.get(1); - replicaParentRegion = allRegionLocations.get(0); -} - -List primaryDaughters = splitRegion(primaryParentRegion.getRegionInfo(), -Bytes.toBytes("a")); - -// Wait until the replica parent region is offline. -while (am.getRegionStates().isRegionOnline(replicaParentRegion.getRegionInfo())) { - Thread.sleep(100); -} - -assertNotNull("Should have found daughter regions for " + primaryDaughters, primaryDaughters); - -// check that primary parent region is not in AM's serverHoldings -assertFalse("Primary Parent region should have been removed from RegionState's serverHoldings", - am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(), -primaryParentRegion.getRegionInfo())); - -// check that primary parent region is not in AM's serverHoldings -assertFalse("Primary Parent region should have been removed from RegionState's serverHoldings", - am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(), -replicaParentRegion.getRegionInfo())); +final HRegionLocation primaryParentRegion + = RegionReplicaUtil.isDefaultReplica( + allRegionLocations.get(0).getRegionInfo().getReplicaId()) ? allRegionLocations.get(0) +: allRegionLocations.get(1); +final HRegionLocation replicaParentRegion + = RegionReplicaUtil.isDefaultReplica( + allRegionLocations.get(0).getRegionInfo().getReplicaId()) ? allRegionLocations.get(1) + : allRegionLocations.get(0); + +assertNotNull("Should have found daughter regions for " + primaryParentRegion, + splitRegion(primaryParentRegion.getRegionInfo(), Bytes.toBytes("a"))); + +TEST_UTIL.waitFor(60 * 1000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { +return !am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(), + primaryParentRegion.getRegionInfo()) && + !am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(), +replicaParentRegion.getRegionInfo()); + } +}); } /*
[3/6] hbase git commit: HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky
HBASE-20016 TestCatalogJanitorInMemoryStates#testInMemoryForReplicaParentCleanup is flaky (cherry picked from commit 08b993997404aad7ec8f66b8c2b2083062c9ef1a) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f96c356 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f96c356 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f96c356 Branch: refs/heads/branch-1.3 Commit: 5f96c356b8dddaf20f8df3207117400999097127 Parents: 5ed1f41 Author: Chia-Ping Tsai Authored: Sun Feb 18 21:22:58 2018 +0800 Committer: Sean Busbey Committed: Mon Feb 26 09:47:31 2018 -0600 -- .../TestCatalogJanitorInMemoryStates.java | 52 +--- 1 file changed, 22 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5f96c356/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java index 6ea7e2f..40396ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java @@ -124,7 +124,7 @@ public class TestCatalogJanitorInMemoryStates { * AM's serverHoldings and */ @Test(timeout = 18) - public void testInMemoryForReplicaParentCleanup() throws IOException, InterruptedException { + public void testInMemoryForReplicaParentCleanup() throws Exception { final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); final CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().catalogJanitorChore; @@ -139,35 +139,27 @@ public class TestCatalogJanitorInMemoryStates { // There are two regions, one for primary, one for the replica. assertTrue(allRegionLocations.size() == 2); -HRegionLocation replicaParentRegion, primaryParentRegion; -if (RegionReplicaUtil.isDefaultReplica( -allRegionLocations.get(0).getRegionInfo().getReplicaId())) { - primaryParentRegion = allRegionLocations.get(0); - replicaParentRegion = allRegionLocations.get(1); -} else { - primaryParentRegion = allRegionLocations.get(1); - replicaParentRegion = allRegionLocations.get(0); -} - -List primaryDaughters = splitRegion(primaryParentRegion.getRegionInfo(), -Bytes.toBytes("a")); - -// Wait until the replica parent region is offline. -while (am.getRegionStates().isRegionOnline(replicaParentRegion.getRegionInfo())) { - Thread.sleep(100); -} - -assertNotNull("Should have found daughter regions for " + primaryDaughters, primaryDaughters); - -// check that primary parent region is not in AM's serverHoldings -assertFalse("Primary Parent region should have been removed from RegionState's serverHoldings", - am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(), -primaryParentRegion.getRegionInfo())); - -// check that primary parent region is not in AM's serverHoldings -assertFalse("Primary Parent region should have been removed from RegionState's serverHoldings", - am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(), -replicaParentRegion.getRegionInfo())); +final HRegionLocation primaryParentRegion + = RegionReplicaUtil.isDefaultReplica( + allRegionLocations.get(0).getRegionInfo().getReplicaId()) ? allRegionLocations.get(0) +: allRegionLocations.get(1); +final HRegionLocation replicaParentRegion + = RegionReplicaUtil.isDefaultReplica( + allRegionLocations.get(0).getRegionInfo().getReplicaId()) ? allRegionLocations.get(1) + : allRegionLocations.get(0); + +assertNotNull("Should have found daughter regions for " + primaryParentRegion, + splitRegion(primaryParentRegion.getRegionInfo(), Bytes.toBytes("a"))); + +TEST_UTIL.waitFor(60 * 1000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { +return !am.getRegionStates().existsInServerHoldings(primaryParentRegion.getServerName(), + primaryParentRegion.getRegionInfo()) && + !am.getRegionStates().existsInServerHoldings(replicaParentRegion.getServerName(), +replicaParentRegion.getRegionInfo()); + } +}); } /*
[4/6] hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM (cherry picked from commit aaece0ba5e399f248c8255fe509cdb1a862bf299) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/688d4dec Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/688d4dec Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/688d4dec Branch: refs/heads/branch-1.2 Commit: 688d4dec74d16f7641a347f3dcd7aa4fa686758d Parents: d2fe2a5 Author: Esteban Gutierrez Authored: Fri Aug 11 12:56:20 2017 -0500 Committer: Sean Busbey Committed: Mon Feb 26 10:46:44 2018 -0600 -- .../hadoop/hbase/master/CatalogJanitor.java | 4 + .../hadoop/hbase/master/RegionStates.java | 6 + .../hadoop/hbase/master/ServerManager.java | 7 + .../TestCatalogJanitorInMemoryStates.java | 188 +++ 4 files changed, 205 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index 48105c8..893b108 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -216,6 +216,8 @@ public class CatalogJanitor extends ScheduledChore { HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB); MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion); + services.getAssignmentManager().getRegionStates().deleteRegion(regionA); + services.getAssignmentManager().getRegionStates().deleteRegion(regionB); services.getServerManager().removeRegion(regionA); services.getServerManager().removeRegion(regionB); return true; @@ -350,6 +352,8 @@ public class CatalogJanitor extends ScheduledChore { if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); MetaTableAccessor.deleteRegion(this.connection, parent); + if (services.getAssignmentManager().getRegionStates() != null) +services.getAssignmentManager().getRegionStates().deleteRegion(parent); services.getServerManager().removeRegion(parent); result = true; } http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index bfdfecc..cf68a02 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -848,6 +848,12 @@ public class RegionStates { } } + @VisibleForTesting + public boolean isRegionInRegionStates(final HRegionInfo hri) { +return (getRegionState(hri) != null || isRegionOnline(hri)) || isRegionInTransition(hri) +|| isRegionInState(hri, State.OFFLINE, State.CLOSED); + } + /** * Checking if a region was assigned to a server which is not online now. * If so, we should hold re-assign this region till SSH has split its wals. http://git-wip-us.apache.org/repos/asf/hbase/blob/688d4dec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index a420705..7527af7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -1222,6 +1222,13 @@ public class ServerManager { flushedSequenceIdByRegion.remove(encodedName); } + @VisibleForTesting + public boolean isRegionInServerManagerStates(final HRegionInfo hri) { +final byte[] encodedName = hri.getEncodedNameAsBytes(); +return (storeFlushedSequenceIdsByRegion.containsKey(encodedName) +|| flushedSequenceIdByRegion.containsKey(encodedName)); + } + /** * Called by delete table and similar to notify the ServerManager that a region was removed. */ http://git-wip-us.apache.org/repos/a
[2/6] hbase git commit: HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap.
HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap. Signed-off-by: ramkrishna.s.vasudevan (cherry picked from commit 80d183a707c6d0203da1565f6a7ca3edf4bb2291) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ed1f416 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ed1f416 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ed1f416 Branch: refs/heads/branch-1.3 Commit: 5ed1f4166e7f7afb6e5625f5cecd5e045301c15d Parents: 4b44c90 Author: huaxiang sun Authored: Wed Jan 3 12:24:05 2018 -0800 Committer: Sean Busbey Committed: Mon Feb 26 09:47:26 2018 -0600 -- .../hadoop/hbase/master/RegionStates.java | 33 ++- .../TestCatalogJanitorInMemoryStates.java | 91 +--- 2 files changed, 112 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5ed1f416/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 406f5e6..3d0c840 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -555,6 +555,19 @@ public class RegionStates { } /** + * Used in some unit tests + */ + @VisibleForTesting + synchronized boolean existsInServerHoldings(final ServerName serverName, + final HRegionInfo hri) { +Set oldRegions = serverHoldings.get(serverName); +if (oldRegions != null) { + return oldRegions.contains(hri); +} +return false; + } + + /** * A dead server's wals have been split so that all the regions * used to be open on it can be safely assigned now. Mark them assignable. */ @@ -623,8 +636,26 @@ public class RegionStates { deleteRegion(hri); return; } + +/* + * One tricky case, if region here is a replica region and its parent is at + * SPLIT state, its newState should be same as its parent, not OFFLINE. + */ State newState = - expectedState == null ? State.OFFLINE : expectedState; +expectedState == null ? State.OFFLINE : expectedState; + +if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) { + RegionState primateState = getRegionState( + RegionReplicaUtil.getRegionInfoForDefaultReplica(hri)); + if ((primateState != null) && (primateState.getState() == State.SPLIT)) { +if (LOG.isDebugEnabled()) { + LOG.debug("Update region " + hri + "to SPLIT, from primary region " + + RegionReplicaUtil.getRegionInfoForDefaultReplica(hri)); +} +newState = State.SPLIT; + } +} + updateRegionState(hri, newState); String encodedName = hri.getEncodedName(); synchronized (this) { http://git-wip-us.apache.org/repos/asf/hbase/blob/5ed1f416/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java index d2bed9b..6ea7e2f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java @@ -18,21 +18,20 @@ */ package org.apache.hadoop.hbase.master; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.junit.AfterClass; @@ -44,12 +43,6 @@ import org.junit.experimental.categories.Cate
[1/6] hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Repository: hbase Updated Branches: refs/heads/branch-1.2 d2fe2a5a9 -> 41cd37cff refs/heads/branch-1.3 1a36db146 -> 5f96c356b HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM (cherry picked from commit aaece0ba5e399f248c8255fe509cdb1a862bf299) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b44c902 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b44c902 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b44c902 Branch: refs/heads/branch-1.3 Commit: 4b44c9025fd21b1aab10579c5c4c169003c983a8 Parents: 1a36db1 Author: Esteban Gutierrez Authored: Fri Aug 11 12:56:20 2017 -0500 Committer: Sean Busbey Committed: Mon Feb 26 09:47:18 2018 -0600 -- .../hadoop/hbase/master/CatalogJanitor.java | 4 + .../hadoop/hbase/master/RegionStates.java | 6 + .../hadoop/hbase/master/ServerManager.java | 7 + .../TestCatalogJanitorInMemoryStates.java | 188 +++ 4 files changed, 205 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index 48105c8..893b108 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -216,6 +216,8 @@ public class CatalogJanitor extends ScheduledChore { HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB); MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion); + services.getAssignmentManager().getRegionStates().deleteRegion(regionA); + services.getAssignmentManager().getRegionStates().deleteRegion(regionB); services.getServerManager().removeRegion(regionA); services.getServerManager().removeRegion(regionB); return true; @@ -350,6 +352,8 @@ public class CatalogJanitor extends ScheduledChore { if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent); HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent); MetaTableAccessor.deleteRegion(this.connection, parent); + if (services.getAssignmentManager().getRegionStates() != null) +services.getAssignmentManager().getRegionStates().deleteRegion(parent); services.getServerManager().removeRegion(parent); result = true; } http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 9fda8da..406f5e6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -858,6 +858,12 @@ public class RegionStates { } } + @VisibleForTesting + public boolean isRegionInRegionStates(final HRegionInfo hri) { +return (getRegionState(hri) != null || isRegionOnline(hri)) || isRegionInTransition(hri) +|| isRegionInState(hri, State.OFFLINE, State.CLOSED); + } + /** * Checking if a region was assigned to a server which is not online now. * If so, we should hold re-assign this region till SSH has split its wals. http://git-wip-us.apache.org/repos/asf/hbase/blob/4b44c902/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 7ea5fd7..c814c2b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -1223,6 +1223,13 @@ public class ServerManager { flushedSequenceIdByRegion.remove(encodedName); } + @VisibleForTesting + public boolean isRegionInServerManagerStates(final HRegionInfo hri) { +final byte[] encodedName = hri.getEncodedNameAsBytes(); +return (storeFlushedSequenceIdsByRegion.containsKey(encodedName) +|| flushedSequenceIdByRegion.containsKey(encodedName)); + } + /** * Cal
[5/6] hbase git commit: HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap.
HBASE-18625 Splitting of region with replica, doesn't update region list in serverHolding. A server crash leads to overlap. Signed-off-by: ramkrishna.s.vasudevan (cherry picked from commit 80d183a707c6d0203da1565f6a7ca3edf4bb2291) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b795553 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b795553 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b795553 Branch: refs/heads/branch-1.2 Commit: 4b795553ea5a05d16088db08c9cc4994e354d56c Parents: 688d4de Author: huaxiang sun Authored: Wed Jan 3 12:24:05 2018 -0800 Committer: Sean Busbey Committed: Mon Feb 26 10:46:54 2018 -0600 -- .../hadoop/hbase/master/RegionStates.java | 33 ++- .../TestCatalogJanitorInMemoryStates.java | 91 +--- 2 files changed, 112 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4b795553/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index cf68a02..3b89f7b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -545,6 +545,19 @@ public class RegionStates { } /** + * Used in some unit tests + */ + @VisibleForTesting + synchronized boolean existsInServerHoldings(final ServerName serverName, + final HRegionInfo hri) { +Set oldRegions = serverHoldings.get(serverName); +if (oldRegions != null) { + return oldRegions.contains(hri); +} +return false; + } + + /** * A dead server's wals have been split so that all the regions * used to be open on it can be safely assigned now. Mark them assignable. */ @@ -613,8 +626,26 @@ public class RegionStates { deleteRegion(hri); return; } + +/* + * One tricky case, if region here is a replica region and its parent is at + * SPLIT state, its newState should be same as its parent, not OFFLINE. + */ State newState = - expectedState == null ? State.OFFLINE : expectedState; +expectedState == null ? State.OFFLINE : expectedState; + +if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) { + RegionState primateState = getRegionState( + RegionReplicaUtil.getRegionInfoForDefaultReplica(hri)); + if ((primateState != null) && (primateState.getState() == State.SPLIT)) { +if (LOG.isDebugEnabled()) { + LOG.debug("Update region " + hri + "to SPLIT, from primary region " + + RegionReplicaUtil.getRegionInfoForDefaultReplica(hri)); +} +newState = State.SPLIT; + } +} + updateRegionState(hri, newState); String encodedName = hri.getEncodedName(); synchronized (this) { http://git-wip-us.apache.org/repos/asf/hbase/blob/4b795553/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java index d2bed9b..6ea7e2f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java @@ -18,21 +18,20 @@ */ package org.apache.hadoop.hbase.master; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.junit.AfterClass; @@ -44,12 +43,6 @@ import org.junit.experimental.categories.Cate
hbase git commit: for creating patch HBASE-20074-V01.patch
Repository: hbase Updated Branches: refs/heads/branch-2 372c68fca -> a312705db for creating patch HBASE-20074-V01.patch Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a312705d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a312705d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a312705d Branch: refs/heads/branch-2 Commit: a312705dbc8e6d604adcc874526294c72b8ff580 Parents: 372c68f Author: gsheffi Authored: Mon Feb 26 11:18:38 2018 +0200 Committer: Michael Stack Committed: Mon Feb 26 09:56:44 2018 -0800 -- .../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++--- 1 file changed, 3 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a312705d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 44b40eb..d60b049 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore { numStores = 1; } inmemoryFlushSize = memstoreFlushSize / numStores; -// multiply by a factor (different factors for different index types) -if (indexType == IndexType.ARRAY_MAP) { - factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, +// multiply by a factor (the same factor for all index types) +factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); -} else { - factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, - IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); -} + inmemoryFlushSize = (long) (inmemoryFlushSize * factor); LOG.info("Setting in-memory flush size threshold to {} and immutable segments index to type={}", StringUtils.byteDesc(inmemoryFlushSize), indexType);
hbase git commit: for creating patch HBASE-20074-V01.patch
Repository: hbase Updated Branches: refs/heads/master 7cfb46432 -> 73028d5bd for creating patch HBASE-20074-V01.patch Signed-off-by: Michael Stack Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73028d5b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73028d5b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73028d5b Branch: refs/heads/master Commit: 73028d5bd9f85655b284654579ddcbbca31e41e8 Parents: 7cfb464 Author: gsheffi Authored: Mon Feb 26 11:18:38 2018 +0200 Committer: Michael Stack Committed: Mon Feb 26 09:57:22 2018 -0800 -- .../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++--- 1 file changed, 3 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/73028d5b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 44b40eb..d60b049 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore { numStores = 1; } inmemoryFlushSize = memstoreFlushSize / numStores; -// multiply by a factor (different factors for different index types) -if (indexType == IndexType.ARRAY_MAP) { - factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, +// multiply by a factor (the same factor for all index types) +factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); -} else { - factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, - IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); -} + inmemoryFlushSize = (long) (inmemoryFlushSize * factor); LOG.info("Setting in-memory flush size threshold to {} and immutable segments index to type={}", StringUtils.byteDesc(inmemoryFlushSize), indexType);
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server
Repository: hbase Updated Branches: refs/heads/branch-2 a312705db -> 44544c7db HBASE-20069 fix existing findbugs errors in hbase-server Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/44544c7d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/44544c7d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/44544c7d Branch: refs/heads/branch-2 Commit: 44544c7db0069683be6e7d8d6adfcdd2a8c40049 Parents: a312705 Author: Michael Stack Authored: Sat Feb 24 13:01:02 2018 -0800 Committer: Michael Stack Committed: Mon Feb 26 10:55:53 2018 -0800 -- .../hbase/io/encoding/EncodedDataBlock.java | 19 +-- .../apache/hadoop/hbase/nio/MultiByteBuff.java | 4 +- .../hadoop/hbase/nio/TestMultiByteBuff.java | 19 +++ .../hbase/procedure2/ProcedureExecutor.java | 1 - .../hbase/procedure2/StateMachineProcedure.java | 1 - .../org/apache/hadoop/hbase/ipc/RpcServer.java | 3 + .../org/apache/hadoop/hbase/master/HMaster.java | 3 - .../master/assignment/AssignmentManager.java| 1 - .../assignment/SplitTableRegionProcedure.java | 7 +- .../hbase/master/cleaner/CleanerChore.java | 39 --- .../hadoop/hbase/regionserver/HRegion.java | 3 +- .../hbase/regionserver/MemStoreFlusher.java | 115 +-- .../hbase/regionserver/RSRpcServices.java | 1 - .../regionserver/RegionCoprocessorHost.java | 2 + .../hbase/regionserver/wal/AsyncFSWAL.java | 4 +- .../hbase/util/compaction/MajorCompactor.java | 9 +- 16 files changed, 147 insertions(+), 84 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/44544c7d/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java index a791c09..af68656 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java @@ -228,6 +228,7 @@ public class EncodedDataBlock { */ public byte[] encodeData() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); +byte [] baosBytes = null; try { baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream out = new DataOutputStream(baos); @@ -255,25 +256,17 @@ public class EncodedDataBlock { kv.setSequenceId(memstoreTS); this.dataBlockEncoder.encode(kv, encodingCtx, out); } - BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream(); - baos.writeTo(stream); - this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, stream.ourBytes); + // Below depends on BAOS internal behavior. toByteArray makes a copy of bytes so far. + baos.flush(); + baosBytes = baos.toByteArray(); + this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, baosBytes); } catch (IOException e) { throw new RuntimeException(String.format( "Bug in encoding part of algorithm %s. " + "Probably it requested more bytes than are available.", toString()), e); } -return baos.toByteArray(); - } - - private static class BufferGrabbingByteArrayOutputStream extends ByteArrayOutputStream { -private byte[] ourBytes; - -@Override -public synchronized void write(byte[] b, int off, int len) { - this.ourBytes = b; -} +return baosBytes; } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/44544c7d/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java index fecf012..847e2eb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java @@ -282,7 +282,7 @@ public class MultiByteBuff extends ByteBuff { return ByteBufferUtils.toShort(item, offsetInItem); } if (items.length - 1 == itemIndex) { - // means cur item is the last one and we wont be able to read a int. Throw exception + // means cur item is the last one and we wont be able to read a short. Throw exception throw new BufferUnderflowException(); } ByteBuffer nextItem = items[itemIndex + 1]; @@ -294,7 +294,7 @@ public class MultiByteBuff extends ByteBuff { } for (int i = 0; i < Bytes.SIZEOF_SHORT - re
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server
Repository: hbase Updated Branches: refs/heads/master 73028d5bd -> b11e50666 HBASE-20069 fix existing findbugs errors in hbase-server Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b11e5066 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b11e5066 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b11e5066 Branch: refs/heads/master Commit: b11e506664614c243c08949c256430d4dd13ba6c Parents: 73028d5 Author: Michael Stack Authored: Sat Feb 24 13:01:02 2018 -0800 Committer: Michael Stack Committed: Mon Feb 26 16:01:31 2018 -0800 -- .../hbase/io/encoding/EncodedDataBlock.java | 19 +-- .../apache/hadoop/hbase/nio/MultiByteBuff.java | 4 +- .../hadoop/hbase/nio/TestMultiByteBuff.java | 19 +++ .../hbase/procedure2/ProcedureExecutor.java | 1 - .../hbase/procedure2/StateMachineProcedure.java | 1 - .../org/apache/hadoop/hbase/ipc/RpcServer.java | 3 + .../org/apache/hadoop/hbase/master/HMaster.java | 1 - .../master/assignment/AssignmentManager.java| 1 - .../assignment/SplitTableRegionProcedure.java | 7 +- .../hbase/master/cleaner/CleanerChore.java | 39 --- .../hadoop/hbase/regionserver/HRegion.java | 3 +- .../hbase/regionserver/MemStoreFlusher.java | 115 +-- .../hbase/regionserver/RSRpcServices.java | 1 - .../regionserver/RegionCoprocessorHost.java | 2 + .../hbase/regionserver/wal/AsyncFSWAL.java | 4 +- .../hbase/util/compaction/MajorCompactor.java | 9 +- 16 files changed, 147 insertions(+), 82 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java index a791c09..af68656 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java @@ -228,6 +228,7 @@ public class EncodedDataBlock { */ public byte[] encodeData() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); +byte [] baosBytes = null; try { baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream out = new DataOutputStream(baos); @@ -255,25 +256,17 @@ public class EncodedDataBlock { kv.setSequenceId(memstoreTS); this.dataBlockEncoder.encode(kv, encodingCtx, out); } - BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream(); - baos.writeTo(stream); - this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, stream.ourBytes); + // Below depends on BAOS internal behavior. toByteArray makes a copy of bytes so far. + baos.flush(); + baosBytes = baos.toByteArray(); + this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, baosBytes); } catch (IOException e) { throw new RuntimeException(String.format( "Bug in encoding part of algorithm %s. " + "Probably it requested more bytes than are available.", toString()), e); } -return baos.toByteArray(); - } - - private static class BufferGrabbingByteArrayOutputStream extends ByteArrayOutputStream { -private byte[] ourBytes; - -@Override -public synchronized void write(byte[] b, int off, int len) { - this.ourBytes = b; -} +return baosBytes; } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java -- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java index fecf012..847e2eb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java @@ -282,7 +282,7 @@ public class MultiByteBuff extends ByteBuff { return ByteBufferUtils.toShort(item, offsetInItem); } if (items.length - 1 == itemIndex) { - // means cur item is the last one and we wont be able to read a int. Throw exception + // means cur item is the last one and we wont be able to read a short. Throw exception throw new BufferUnderflowException(); } ByteBuffer nextItem = items[itemIndex + 1]; @@ -294,7 +294,7 @@ public class MultiByteBuff extends ByteBuff { } for (int i = 0; i < Bytes.SIZEOF_SHORT - remain
hbase git commit: HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region
Repository: hbase Updated Branches: refs/heads/branch-1.4 8d3fc596c -> afe892fd4 HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/afe892fd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/afe892fd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/afe892fd Branch: refs/heads/branch-1.4 Commit: afe892fd4d31a2bfe9c8ed903d48adac8e4ce8d5 Parents: 8d3fc59 Author: Thiruvel Thirumoolan Authored: Mon Feb 26 11:35:09 2018 -0800 Committer: tedyu Committed: Mon Feb 26 18:01:57 2018 -0800 -- .../hadoop/hbase/master/AssignmentManager.java | 32 +++- .../hadoop/hbase/master/RegionStates.java | 42 +++-- .../TestRegionMergeTransactionOnCluster.java| 183 ++- .../TestSplitTransactionOnCluster.java | 134 -- 4 files changed, 316 insertions(+), 75 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/afe892fd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index c7339ef..bc1e02f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -255,10 +255,10 @@ public class AssignmentManager extends ZooKeeperListener { private final RegionStateStore regionStateStore; /** - * For testing only! Set to true to skip handling of split. + * For testing only! Set to true to skip handling of split and merge. */ - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL") - public static boolean TEST_SKIP_SPLIT_HANDLING = false; + private static boolean TEST_SKIP_SPLIT_HANDLING = false; + private static boolean TEST_SKIP_MERGE_HANDLING = false; /** Listeners that are called on assignment events. */ private List listeners = new CopyOnWriteArrayList(); @@ -4054,6 +4054,11 @@ public class AssignmentManager extends ZooKeeperListener { mergingRegions.put(encodedName, new PairOfSameType(a, b)); } else if (code == TransitionCode.MERGED) { + + if (TEST_SKIP_MERGE_HANDLING) { +return "Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p; + } + mergingRegions.remove(encodedName); regionOffline(a, State.MERGED); regionOffline(b, State.MERGED); @@ -4180,6 +4185,11 @@ public class AssignmentManager extends ZooKeeperListener { regionStates.updateRegionState(hri_b, State.MERGING); regionStates.updateRegionState(p, State.MERGING_NEW, sn); + if (TEST_SKIP_MERGE_HANDLING) { +LOG.warn("Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p); +return true; // return true so that the merging node stays + } + if (et != EventType.RS_ZK_REGION_MERGED) { this.mergingRegions.put(encodedName, new PairOfSameType(hri_a, hri_b)); @@ -4727,4 +4737,20 @@ public class AssignmentManager extends ZooKeeperListener { threadPoolExecutorService.submit(callable); } } + + /* + * This is only used for unit-testing split failures. + */ + @VisibleForTesting + public static void setTestSkipSplitHandling(boolean skipSplitHandling) { +TEST_SKIP_SPLIT_HANDLING = skipSplitHandling; + } + + /* + * This is only used for unit-testing merge failures. + */ + @VisibleForTesting + public static void setTestSkipMergeHandling(boolean skipMergeHandling) { +TEST_SKIP_MERGE_HANDLING = skipMergeHandling; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/afe892fd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 7ce6257..f47d555 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.
hbase git commit: HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region
Repository: hbase Updated Branches: refs/heads/branch-1 a37c91bb4 -> ece68d3c6 HBASE-20001 cleanIfNoMetaEntry() uses encoded instead of region name to lookup region Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ece68d3c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ece68d3c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ece68d3c Branch: refs/heads/branch-1 Commit: ece68d3c650381e0b7981b685f3e9f2f48b5dec5 Parents: a37c91b Author: Thiruvel Thirumoolan Authored: Mon Feb 26 11:35:09 2018 -0800 Committer: tedyu Committed: Mon Feb 26 18:01:28 2018 -0800 -- .../hadoop/hbase/master/AssignmentManager.java | 32 +++- .../hadoop/hbase/master/RegionStates.java | 42 +++-- .../TestRegionMergeTransactionOnCluster.java| 183 ++- .../TestSplitTransactionOnCluster.java | 134 -- 4 files changed, 316 insertions(+), 75 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/ece68d3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index c7339ef..bc1e02f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -255,10 +255,10 @@ public class AssignmentManager extends ZooKeeperListener { private final RegionStateStore regionStateStore; /** - * For testing only! Set to true to skip handling of split. + * For testing only! Set to true to skip handling of split and merge. */ - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL") - public static boolean TEST_SKIP_SPLIT_HANDLING = false; + private static boolean TEST_SKIP_SPLIT_HANDLING = false; + private static boolean TEST_SKIP_MERGE_HANDLING = false; /** Listeners that are called on assignment events. */ private List listeners = new CopyOnWriteArrayList(); @@ -4054,6 +4054,11 @@ public class AssignmentManager extends ZooKeeperListener { mergingRegions.put(encodedName, new PairOfSameType(a, b)); } else if (code == TransitionCode.MERGED) { + + if (TEST_SKIP_MERGE_HANDLING) { +return "Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p; + } + mergingRegions.remove(encodedName); regionOffline(a, State.MERGED); regionOffline(b, State.MERGED); @@ -4180,6 +4185,11 @@ public class AssignmentManager extends ZooKeeperListener { regionStates.updateRegionState(hri_b, State.MERGING); regionStates.updateRegionState(p, State.MERGING_NEW, sn); + if (TEST_SKIP_MERGE_HANDLING) { +LOG.warn("Skipping merge message, TEST_SKIP_MERGE_HANDLING is set for merge parent: " + p); +return true; // return true so that the merging node stays + } + if (et != EventType.RS_ZK_REGION_MERGED) { this.mergingRegions.put(encodedName, new PairOfSameType(hri_a, hri_b)); @@ -4727,4 +4737,20 @@ public class AssignmentManager extends ZooKeeperListener { threadPoolExecutorService.submit(callable); } } + + /* + * This is only used for unit-testing split failures. + */ + @VisibleForTesting + public static void setTestSkipSplitHandling(boolean skipSplitHandling) { +TEST_SKIP_SPLIT_HANDLING = skipSplitHandling; + } + + /* + * This is only used for unit-testing merge failures. + */ + @VisibleForTesting + public static void setTestSkipMergeHandling(boolean skipMergeHandling) { +TEST_SKIP_MERGE_HANDLING = skipMergeHandling; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/ece68d3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 7ce6257..f47d555 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbas
hbase git commit: HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Repository: hbase Updated Branches: refs/heads/master b11e50666 -> d3aefe783 HBASE-20086 PE randomSeekScan fails with ClassNotFoundException Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3aefe78 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3aefe78 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3aefe78 Branch: refs/heads/master Commit: d3aefe783476e860e7b1c474b50cf18a7ae0be00 Parents: b11e506 Author: tedyu Authored: Mon Feb 26 18:29:35 2018 -0800 Committer: tedyu Committed: Mon Feb 26 18:29:35 2018 -0800 -- .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d3aefe78/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java -- diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index a5fa2c7..5a63ef4 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -527,7 +527,9 @@ public class PerformanceEvaluation extends Configured implements Tool { TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Histogram.class, // yammer metrics - ObjectMapper.class); // jackson-mapper-asl + ObjectMapper.class, // jackson-mapper-asl + FilterAllFilter.class // hbase-server tests jar + ); TableMapReduceUtil.initCredentials(job);
hbase git commit: HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Repository: hbase Updated Branches: refs/heads/branch-2 44544c7db -> b3ae58021 HBASE-20086 PE randomSeekScan fails with ClassNotFoundException Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3ae5802 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3ae5802 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3ae5802 Branch: refs/heads/branch-2 Commit: b3ae58021607e44f99ac3b31dbe716e242a45110 Parents: 44544c7 Author: tedyu Authored: Mon Feb 26 18:30:09 2018 -0800 Committer: tedyu Committed: Mon Feb 26 18:30:09 2018 -0800 -- .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b3ae5802/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java -- diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index b2ebaad..95d363f 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -527,7 +527,9 @@ public class PerformanceEvaluation extends Configured implements Tool { TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Histogram.class, // yammer metrics - ObjectMapper.class); // jackson-mapper-asl + ObjectMapper.class, // jackson-mapper-asl + FilterAllFilter.class // hbase-server tests jar + ); TableMapReduceUtil.initCredentials(job);
hbase git commit: HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing.
Repository: hbase Updated Branches: refs/heads/branch-1.4 afe892fd4 -> c5bc25a7d HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing. Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c5bc25a7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c5bc25a7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c5bc25a7 Branch: refs/heads/branch-1.4 Commit: c5bc25a7d759adbb99f30217f97794046f6c7c24 Parents: afe892f Author: Artem Ervits Authored: Mon Feb 26 18:19:21 2018 -0500 Committer: tedyu Committed: Mon Feb 26 19:29:05 2018 -0800 -- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/c5bc25a7/hbase-shaded/hbase-shaded-check-invariants/pom.xml -- diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 7f7eb4f..e77fdfb 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -161,6 +161,7 @@ org.codehaus.mojo exec-maven-plugin +1.6.0 check-jar-contents
hbase git commit: HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing.
Repository: hbase Updated Branches: refs/heads/branch-1 ece68d3c6 -> 8e5b8f22f HBASE-20091 Fix for 'build.plugins.plugin.version' for org.codehaus.mojo: is missing. Signed-off-by: tedyu Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e5b8f22 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e5b8f22 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e5b8f22 Branch: refs/heads/branch-1 Commit: 8e5b8f22fc967705cdc2b3ea724e8713ef3d7eff Parents: ece68d3 Author: Artem Ervits Authored: Mon Feb 26 18:19:21 2018 -0500 Committer: tedyu Committed: Mon Feb 26 19:29:36 2018 -0800 -- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8e5b8f22/hbase-shaded/hbase-shaded-check-invariants/pom.xml -- diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index 01587db..67faaa4 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -161,6 +161,7 @@ org.codehaus.mojo exec-maven-plugin +1.6.0 check-jar-contents
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review
Repository: hbase Updated Branches: refs/heads/branch-2 b3ae58021 -> 43d661b00 HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/43d661b0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/43d661b0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/43d661b0 Branch: refs/heads/branch-2 Commit: 43d661b008cda46604ddde2187945bb0caa5bc5f Parents: b3ae580 Author: Michael Stack Authored: Mon Feb 26 16:26:36 2018 -0800 Committer: Michael Stack Committed: Mon Feb 26 23:15:43 2018 -0800 -- .../hadoop/hbase/master/ClusterStatusPublisher.java | 6 +- .../hadoop/hbase/regionserver/MemStoreFlusher.java| 14 +++--- .../hbase/regionserver/RegionCoprocessorHost.java | 4 +++- 3 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java index 5e97204..21fa263 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java @@ -134,7 +134,7 @@ public class ClusterStatusPublisher extends ScheduledChore { @Override protected void chore() { -if (!connected) { +if (!isConnected()) { return; } @@ -170,6 +170,10 @@ public class ClusterStatusPublisher extends ScheduledChore { publisher.close(); } + private synchronized boolean isConnected() { +return this.connected; + } + /** * Create the dead server to send. A dead server is sent NB_SEND times. We send at max * MAX_SERVER_PER_MESSAGE at a time. if there are too many dead servers, we send the newly http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index a0e65ec..23321e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -90,17 +90,9 @@ class MemStoreFlusher implements FlushRequester { private FlushType flushType; /** - * Singleton instance of this class inserted into flush queue. + * Singleton instance inserted into flush queue used for signaling. */ - private static final WakeupFlushThread WAKEUPFLUSH_INSTANCE = new WakeupFlushThread(); - - /** - * Marker class used as a token inserted into flush queue that ensures the flusher does not sleep. - * Create a single instance only. - */ - private static final class WakeupFlushThread implements FlushQueueEntry { -private WakeupFlushThread() {} - + private static final FlushQueueEntry WAKEUPFLUSH_INSTANCE = new FlushQueueEntry() { @Override public long getDelay(TimeUnit unit) { return 0; @@ -120,7 +112,7 @@ class MemStoreFlusher implements FlushRequester { public int hashCode() { return 42; } - } + }; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/43d661b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index f3c93dc..47b389a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -1119,6 +1119,8 @@ public class RegionCoprocessorHost * @return true or false to return to client if default processing should be bypassed, or null * otherwise */ + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_BOOLEAN_RETURN_NULL", + justification="Null is legit") public Boolean preCheckAndPutAfterRowLock( final byte[] row, final byte[] family, final byte[] qualifier, final CompareOperator op, final ByteArrayComparable comparator, final Put put) throws IOException { @@ -1207,7 +12
hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review
Repository: hbase Updated Branches: refs/heads/master d3aefe783 -> d272ac908 HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d272ac90 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d272ac90 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d272ac90 Branch: refs/heads/master Commit: d272ac908ceb4696e05431066ae02d953fa6fc9d Parents: d3aefe7 Author: Michael Stack Authored: Mon Feb 26 16:26:36 2018 -0800 Committer: Michael Stack Committed: Mon Feb 26 23:16:21 2018 -0800 -- .../hadoop/hbase/master/ClusterStatusPublisher.java | 6 +- .../hadoop/hbase/regionserver/MemStoreFlusher.java| 14 +++--- .../hbase/regionserver/RegionCoprocessorHost.java | 4 +++- 3 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java index 5e97204..21fa263 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java @@ -134,7 +134,7 @@ public class ClusterStatusPublisher extends ScheduledChore { @Override protected void chore() { -if (!connected) { +if (!isConnected()) { return; } @@ -170,6 +170,10 @@ public class ClusterStatusPublisher extends ScheduledChore { publisher.close(); } + private synchronized boolean isConnected() { +return this.connected; + } + /** * Create the dead server to send. A dead server is sent NB_SEND times. We send at max * MAX_SERVER_PER_MESSAGE at a time. if there are too many dead servers, we send the newly http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index a0e65ec..23321e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -90,17 +90,9 @@ class MemStoreFlusher implements FlushRequester { private FlushType flushType; /** - * Singleton instance of this class inserted into flush queue. + * Singleton instance inserted into flush queue used for signaling. */ - private static final WakeupFlushThread WAKEUPFLUSH_INSTANCE = new WakeupFlushThread(); - - /** - * Marker class used as a token inserted into flush queue that ensures the flusher does not sleep. - * Create a single instance only. - */ - private static final class WakeupFlushThread implements FlushQueueEntry { -private WakeupFlushThread() {} - + private static final FlushQueueEntry WAKEUPFLUSH_INSTANCE = new FlushQueueEntry() { @Override public long getDelay(TimeUnit unit) { return 0; @@ -120,7 +112,7 @@ class MemStoreFlusher implements FlushRequester { public int hashCode() { return 42; } - } + }; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index f3c93dc..47b389a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -1119,6 +1119,8 @@ public class RegionCoprocessorHost * @return true or false to return to client if default processing should be bypassed, or null * otherwise */ + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_BOOLEAN_RETURN_NULL", + justification="Null is legit") public Boolean preCheckAndPutAfterRowLock( final byte[] row, final byte[] family, final byte[] qualifier, final CompareOperator op, final ByteArrayComparable comparator, final Put put) throws IOException { @@ -1207,7 +1209,7
hbase git commit: HBASE-20066 Region sequence id may go backward after split or merge
Repository: hbase Updated Branches: refs/heads/master d272ac908 -> f06a89b53 HBASE-20066 Region sequence id may go backward after split or merge Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f06a89b5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f06a89b5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f06a89b5 Branch: refs/heads/master Commit: f06a89b5317a8e3d4fb220eb462b083bc6e0b5c8 Parents: d272ac9 Author: zhangduo Authored: Mon Feb 26 20:22:57 2018 +0800 Committer: zhangduo Committed: Tue Feb 27 15:33:07 2018 +0800 -- .../src/main/protobuf/MasterProcedure.proto | 22 ++- .../assignment/MergeTableRegionsProcedure.java | 181 +-- .../assignment/SplitTableRegionProcedure.java | 150 +++ .../AbstractStateMachineTableProcedure.java | 14 +- .../hadoop/hbase/regionserver/HRegion.java | 18 +- .../apache/hadoop/hbase/wal/WALSplitter.java| 109 ++- .../TestSequenceIdMonotonicallyIncreasing.java | 156 .../hadoop/hbase/master/AbstractTestDLS.java| 36 .../hbase/wal/TestReadWriteSeqIdFiles.java | 95 ++ 9 files changed, 504 insertions(+), 277 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 1ab51e5..e785f96 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -241,11 +241,12 @@ enum SplitTableRegionState { SPLIT_TABLE_REGION_PRE_OPERATION = 2; SPLIT_TABLE_REGION_CLOSE_PARENT_REGION = 3; SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS = 4; - SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 5; - SPLIT_TABLE_REGION_UPDATE_META = 6; - SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 7; - SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 8; - SPLIT_TABLE_REGION_POST_OPERATION = 9; + SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE = 5; + SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 6; + SPLIT_TABLE_REGION_UPDATE_META = 7; + SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 8; + SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 9; + SPLIT_TABLE_REGION_POST_OPERATION = 10; } message SplitTableRegionStateData { @@ -260,11 +261,12 @@ enum MergeTableRegionsState { MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION = 3; MERGE_TABLE_REGIONS_CLOSE_REGIONS = 4; MERGE_TABLE_REGIONS_CREATE_MERGED_REGION = 5; - MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 6; - MERGE_TABLE_REGIONS_UPDATE_META = 7; - MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 8; - MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 9; - MERGE_TABLE_REGIONS_POST_OPERATION = 10; + MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE = 6; + MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 7; + MERGE_TABLE_REGIONS_UPDATE_META = 8; + MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 9; + MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 10; + MERGE_TABLE_REGIONS_POST_OPERATION = 11; } message MergeTableRegionsStateData { http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 1c448dc..7c041e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -204,55 +205,55 @@ public class MergeTableRegionsProcedure } @Override - protected Flow executeFromState( - final MasterProcedureEnv env, - final MergeTableRegionsState state) - throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { -if (LOG.isDebugEnabled()) { - LOG.debug(this + " execute state=" + state); -} + protected Flow executeFromState(final MasterProcedureEnv env, final MergeTableRegionsState state)
hbase git commit: HBASE-20066 Region sequence id may go backward after split or merge
Repository: hbase Updated Branches: refs/heads/branch-2 43d661b00 -> 8e8e50683 HBASE-20066 Region sequence id may go backward after split or merge Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e8e5068 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e8e5068 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e8e5068 Branch: refs/heads/branch-2 Commit: 8e8e50683d23124c4a8dbcb06a338dfa03ca7f85 Parents: 43d661b Author: zhangduo Authored: Mon Feb 26 20:22:57 2018 +0800 Committer: zhangduo Committed: Tue Feb 27 15:37:32 2018 +0800 -- .../src/main/protobuf/MasterProcedure.proto | 22 ++- .../assignment/MergeTableRegionsProcedure.java | 181 +-- .../assignment/SplitTableRegionProcedure.java | 150 +++ .../AbstractStateMachineTableProcedure.java | 14 +- .../hadoop/hbase/regionserver/HRegion.java | 18 +- .../apache/hadoop/hbase/wal/WALSplitter.java| 103 +-- .../TestSequenceIdMonotonicallyIncreasing.java | 156 .../hadoop/hbase/master/AbstractTestDLS.java| 36 .../hbase/wal/TestReadWriteSeqIdFiles.java | 95 ++ 9 files changed, 501 insertions(+), 274 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8e8e5068/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto -- diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index f988224..5b42ca9 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -240,11 +240,12 @@ enum SplitTableRegionState { SPLIT_TABLE_REGION_PRE_OPERATION = 2; SPLIT_TABLE_REGION_CLOSE_PARENT_REGION = 3; SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS = 4; - SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 5; - SPLIT_TABLE_REGION_UPDATE_META = 6; - SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 7; - SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 8; - SPLIT_TABLE_REGION_POST_OPERATION = 9; + SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE = 5; + SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 6; + SPLIT_TABLE_REGION_UPDATE_META = 7; + SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 8; + SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 9; + SPLIT_TABLE_REGION_POST_OPERATION = 10; } message SplitTableRegionStateData { @@ -259,11 +260,12 @@ enum MergeTableRegionsState { MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION = 3; MERGE_TABLE_REGIONS_CLOSE_REGIONS = 4; MERGE_TABLE_REGIONS_CREATE_MERGED_REGION = 5; - MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 6; - MERGE_TABLE_REGIONS_UPDATE_META = 7; - MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 8; - MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 9; - MERGE_TABLE_REGIONS_POST_OPERATION = 10; + MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE = 6; + MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 7; + MERGE_TABLE_REGIONS_UPDATE_META = 8; + MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 9; + MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 10; + MERGE_TABLE_REGIONS_POST_OPERATION = 11; } message MergeTableRegionsStateData { http://git-wip-us.apache.org/repos/asf/hbase/blob/8e8e5068/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 1c448dc..7c041e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -204,55 +205,55 @@ public class MergeTableRegionsProcedure } @Override - protected Flow executeFromState( - final MasterProcedureEnv env, - final MergeTableRegionsState state) - throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { -if (LOG.isDebugEnabled()) { - LOG.debug(this + " execute state=" + state); -} + protected Flow executeFromState(final MasterProcedureEnv env, final MergeTableRegionsState sta