hbase git commit: HBASE-20050 Reimplement updateReplicationPositions logic in serial replication based on the newly introduced replication storage layer
Repository: hbase
Updated Branches:
refs/heads/master c415ddb38 -> 99d3edfc8
HBASE-20050 Reimplement updateReplicationPositions logic in serial replication
based on the newly introduced replication storage layer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/99d3edfc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/99d3edfc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/99d3edfc
Branch: refs/heads/master
Commit: 99d3edfc82d89d6da23008f3ab0a203317cec6f2
Parents: c415ddb
Author: huzheng
Authored: Wed Feb 28 16:25:24 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:53:14 2018 +0800
--
.../replication/ReplicationQueueStorage.java| 15 +++-
.../replication/ZKReplicationQueueStorage.java | 88 ++--
.../replication/TestReplicationStateBasic.java | 48 ++-
.../TestZKReplicationQueueStorage.java | 7 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
5 files changed, 146 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/99d3edfc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index e774148..4c93da6 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
@@ -63,9 +64,19 @@ public interface ReplicationQueueStorage {
* @param queueId a String that identifies the queue
* @param fileName name of the WAL
* @param position the current position in the file
+ * @param lastSeqIds map with {encodedRegionName, sequenceId} pairs for
serial replication.
*/
- void setWALPosition(ServerName serverName, String queueId, String fileName,
long position)
- throws ReplicationException;
+ void setWALPosition(ServerName serverName, String queueId, String fileName,
long position,
+ Map lastSeqIds) throws ReplicationException;
+
+ /**
+ * Read the max sequence id of the specific region for a given peer. For
serial replication, we
+ * need the max sequenced id to decide whether we can push the next entries.
+ * @param encodedRegionName the encoded region name
+ * @param peerId peer id
+ * @return the max sequence id of the specific region for a given peer.
+ */
+ long getLastSequenceId(String encodedRegionName, String peerId) throws
ReplicationException;
/**
* Get the current position for a specific WAL in a given queue for a given
regionserver.
http://git-wip-us.apache.org/repos/asf/hbase/blob/99d3edfc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
index ee237f2..9a281b1 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -86,6 +88,10 @@ class ZKReplicationQueueStorage extends
ZKReplicationStorageBase
"zookeeper.znode.replication.hfile.refs";
public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT =
"hfile-refs";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY =
+ "zookeeper.znode.replication.regions";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT =
"regions";
+
/**
* The name of the znode that contains all replication queues
*/
@@ -96,6 +102,8 @@ class ZKReplicationQueueStorage extends
ZKReplicationStorageBase
*/
private final String hfileRefsZNode;
+ private final String regionsZNode;
+
public ZKReplicationQueueStorage(ZKWatcher zookeeper, Configuration conf) {
super(zookeeper, conf);
@@ -104,6 +112,8 @@ class ZKRepli
[49/50] [abbrv] hbase git commit: HBASE-19923 Reset peer state and config when refresh replication source failed
HBASE-19923 Reset peer state and config when refresh replication source failed
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3db22e6d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3db22e6d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3db22e6d
Branch: refs/heads/HBASE-19397-branch-2
Commit: 3db22e6dda5d2bb652dde91064ac3d134185b40f
Parents: c1b2ad6
Author: Guanghao Zhang
Authored: Tue Feb 6 14:58:39 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../hbase/replication/ReplicationPeerImpl.java | 4 ++--
.../regionserver/PeerProcedureHandlerImpl.java | 24
2 files changed, 22 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3db22e6d/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 604e0bb..d656466 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -54,11 +54,11 @@ public class ReplicationPeerImpl implements ReplicationPeer
{
this.peerConfigListeners = new ArrayList<>();
}
- void setPeerState(boolean enabled) {
+ public void setPeerState(boolean enabled) {
this.peerState = enabled ? PeerState.ENABLED : PeerState.DISABLED;
}
- void setPeerConfig(ReplicationPeerConfig peerConfig) {
+ public void setPeerConfig(ReplicationPeerConfig peerConfig) {
this.peerConfig = peerConfig;
peerConfigListeners.forEach(listener ->
listener.peerConfigUpdated(peerConfig));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3db22e6d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index ce8fdae..a02d181 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -62,18 +62,26 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private void refreshPeerState(String peerId) throws ReplicationException,
IOException {
PeerState newState;
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+PeerState oldState = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- PeerState oldState = peer.getPeerState();
+ oldState = peer.getPeerState();
newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
// RS need to start work with the new replication state change
if (oldState.equals(PeerState.ENABLED) &&
newState.equals(PeerState.DISABLED)) {
replicationSourceManager.refreshSources(peerId);
}
+ success = true;
} finally {
+ if (!success && peer != null) {
+// Reset peer state if refresh source failed
+peer.setPeerState(oldState.equals(PeerState.ENABLED));
+ }
peerLock.unlock();
}
}
@@ -91,19 +99,27 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
Lock peerLock = peersLock.acquireLock(peerId);
+ReplicationPeerImpl peer = null;
+ReplicationPeerConfig oldConfig = null;
+boolean success = false;
try {
- ReplicationPeerImpl peer =
replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
if (peer == null) {
throw new ReplicationException("Peer with id=" + peerId + " is not
cached.");
}
- ReplicationPeerConfig oldConfig = peer.getPeerConfig();
+ oldConfig = peer.getPeerConfig();
ReplicationPeerConfig newConfig =
repli
[28/50] [abbrv] hbase git commit: HBASE-19623 Create replication endpoint asynchronously when adding a replication source
HBASE-19623 Create replication endpoint asynchronously when adding a
replication source
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ae8a0295
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ae8a0295
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ae8a0295
Branch: refs/heads/HBASE-19397-branch-2
Commit: ae8a029579441aeccc2c7b2b98d8f5bb777d19dd
Parents: 6f9332a
Author: zhangduo
Authored: Tue Jan 2 13:25:58 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../hbase/replication/ReplicationPeer.java | 8 ++
.../hbase/replication/ReplicationPeers.java | 18 +--
.../replication/ZKReplicationPeerStorage.java | 7 +-
.../replication/TestReplicationStateBasic.java | 20 +---
.../TestZKReplicationPeerStorage.java | 14 +--
.../HBaseInterClusterReplicationEndpoint.java | 17 ++-
.../RecoveredReplicationSource.java | 13 +--
.../regionserver/ReplicationSource.java | 110 +++
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 47 +---
.../client/TestAsyncReplicationAdminApi.java| 2 -
.../replication/TestReplicationAdmin.java | 2 -
.../replication/ReplicationSourceDummy.java | 7 +-
.../replication/TestReplicationSource.java | 5 +-
.../TestReplicationSourceManager.java | 8 +-
15 files changed, 116 insertions(+), 170 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae8a0295/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 4846018..2da3cce 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,14 @@ public interface ReplicationPeer {
PeerState getPeerState();
/**
+ * Test whether the peer is enabled.
+ * @return {@code true} if enabled, otherwise {@code false}.
+ */
+ default boolean isPeerEnabled() {
+return getPeerState() == PeerState.ENABLED;
+ }
+
+ /**
* Get the peer config object
* @return the ReplicationPeerConfig for this peer
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae8a0295/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 422801b..45940a5 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -86,21 +87,6 @@ public class ReplicationPeers {
}
/**
- * Get the peer state for the specified connected remote slave cluster. The
value might be read
- * from cache, so it is recommended to use {@link #peerStorage } to read
storage directly if
- * reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- public boolean isPeerEnabled(String peerId) {
-ReplicationPeer replicationPeer = this.peerCache.get(peerId);
-if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + peerId + " is not
cached");
-}
-return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- /**
* Returns the ReplicationPeerImpl for the specified cached peer. This
ReplicationPeer will
* continue to track changes to the Peer's state and config. This method
returns null if no peer
* has been cached with the given peerId.
@@ -117,7 +103,7 @@ public class ReplicationPeers {
* @return a Set of Strings for peerIds
*/
public Set getAllPeerIds() {
-return peerCache.keySet();
+return Collections.unmodifiableSet(peerCache.keySet());
}
public static Configuration
getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
http://git-wip-us.apache.org/repos/asf/hbase/blob/ae8a0295/hbase-
[16/50] [abbrv] hbase git commit: HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure
HBASE-19524 Master side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee7d9b3c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee7d9b3c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee7d9b3c
Branch: refs/heads/HBASE-19397-branch-2
Commit: ee7d9b3cadc977bbd459910b1cc51ffe3dfa2a4e
Parents: 2539260
Author: zhangduo
Authored: Mon Dec 18 15:22:36 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../procedure2/RemoteProcedureDispatcher.java | 3 +-
.../src/main/protobuf/MasterProcedure.proto | 21 -
.../src/main/protobuf/RegionServerStatus.proto | 3 +-
.../src/main/protobuf/Replication.proto | 5 +
.../replication/ReplicationPeersZKImpl.java | 4 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 93 ---
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/master/MasterServices.java | 26 +++---
.../assignment/RegionTransitionProcedure.java | 13 +--
.../master/procedure/MasterProcedureEnv.java| 5 +
.../master/procedure/ProcedurePrepareLatch.java | 2 +-
.../master/replication/AddPeerProcedure.java| 97
.../replication/DisablePeerProcedure.java | 70 ++
.../master/replication/EnablePeerProcedure.java | 69 ++
.../master/replication/ModifyPeerProcedure.java | 97 +---
.../master/replication/RefreshPeerCallable.java | 67 --
.../replication/RefreshPeerProcedure.java | 28 --
.../master/replication/RemovePeerProcedure.java | 69 ++
.../master/replication/ReplicationManager.java | 76 ---
.../replication/UpdatePeerConfigProcedure.java | 92 +++
.../hbase/regionserver/HRegionServer.java | 5 +-
.../regionserver/RefreshPeerCallable.java | 70 ++
.../hbase/master/MockNoopMasterServices.java| 23 +++--
.../replication/DummyModifyPeerProcedure.java | 13 ++-
24 files changed, 733 insertions(+), 222 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee7d9b3c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 7e3dde6..fb852c3 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -247,9 +247,8 @@ public abstract class RemoteProcedureDispatcherhttp://git-wip-us.apache.org/repos/asf/hbase/blob/ee7d9b3c/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 8aa5c78..a9521ad 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -27,6 +27,7 @@ option optimize_for = SPEED;
import "HBase.proto";
import "RPC.proto";
import "Snapshot.proto";
+import "Replication.proto";
//
// WARNING - Compatibility rules
@@ -370,9 +371,10 @@ message GCMergedRegionsStateData {
}
enum PeerModificationState {
- UPDATE_PEER_STORAGE = 1;
- REFRESH_PEER_ON_RS = 2;
- POST_PEER_MODIFICATION = 3;
+ PRE_PEER_MODIFICATION = 1;
+ UPDATE_PEER_STORAGE = 2;
+ REFRESH_PEER_ON_RS = 3;
+ POST_PEER_MODIFICATION = 4;
}
message PeerModificationStateData {
@@ -397,4 +399,17 @@ message RefreshPeerParameter {
required string peer_id = 1;
required PeerModificationType type = 2;
required ServerName target_server = 3;
+}
+
+message ModifyPeerStateData {
+ required string peer_id = 1;
+}
+
+message AddPeerStateData {
+ required ReplicationPeer peer_config = 1;
+ required bool enabled = 2;
+}
+
+message UpdatePeerConfigStateData {
+ required ReplicationPeer peer_config = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee7d9b3c/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index eb396ac..4f75941 100644
--- a/hbase-protocol-sha
[30/50] [abbrv] hbase git commit: HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/84d5e487
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/84d5e487
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/84d5e487
Branch: refs/heads/HBASE-19397-branch-2
Commit: 84d5e487a1849a9f06f591d6d00708da74a757bc
Parents: dbc81ff
Author: zhangduo
Authored: Wed Dec 27 20:13:42 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../src/main/protobuf/RegionServerStatus.proto | 5 +-
.../hadoop/hbase/master/MasterRpcServices.java | 15 ++-
.../hbase/regionserver/HRegionServer.java | 72
.../RemoteProcedureResultReporter.java | 111 +++
.../handler/RSProcedureHandler.java | 2 +-
5 files changed, 149 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/84d5e487/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 4f75941..3f836cd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -146,7 +146,7 @@ message RegionSpaceUseReportRequest {
message RegionSpaceUseReportResponse {
}
-message ReportProcedureDoneRequest {
+message RemoteProcedureResult {
required uint64 proc_id = 1;
enum Status {
SUCCESS = 1;
@@ -155,6 +155,9 @@ message ReportProcedureDoneRequest {
required Status status = 2;
optional ForeignExceptionMessage error = 3;
}
+message ReportProcedureDoneRequest {
+ repeated RemoteProcedureResult result = 1;
+}
message ReportProcedureDoneResponse {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/84d5e487/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 6ce0d39..8f92041 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -268,6 +268,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
@@ -2236,12 +2237,14 @@ public class MasterRpcServices extends RSRpcServices
@Override
public ReportProcedureDoneResponse reportProcedureDone(RpcController
controller,
ReportProcedureDoneRequest request) throws ServiceException {
-if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
- master.remoteProcedureCompleted(request.getProcId());
-} else {
- master.remoteProcedureFailed(request.getProcId(),
-RemoteProcedureException.fromProto(request.getError()));
-}
+request.getResultList().forEach(result -> {
+ if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) {
+master.remoteProcedureCompleted(result.getProcId());
+ } else {
+master.remoteProcedureFailed(result.getProcId(),
+ RemoteProcedureException.fromProto(result.getError()));
+ }
+});
return ReportProcedureDoneResponse.getDefaultInstance();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/84d5e487/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index c95ac37..81febc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regions
[44/50] [abbrv] hbase git commit: HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint
HBASE-19707 Race in start and terminate of a replication source after we async
start replicatione endpoint
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/388fb4bd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/388fb4bd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/388fb4bd
Branch: refs/heads/HBASE-19397-branch-2
Commit: 388fb4bd96ce0edd248a196c1ad8a357263e795f
Parents: d5277a2
Author: zhangduo
Authored: Fri Jan 5 18:28:44 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../RecoveredReplicationSource.java | 16 +-
.../regionserver/ReplicationSource.java | 203 ++-
2 files changed, 116 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/388fb4bd/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 1be9a88..3cae0f2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -68,7 +68,7 @@ public class RecoveredReplicationSource extends
ReplicationSource {
LOG.debug("Someone has beat us to start a worker thread for wal group "
+ walGroupId);
} else {
LOG.debug("Starting up worker for wal group " + walGroupId);
- worker.startup(getUncaughtExceptionHandler());
+ worker.startup(this::uncaughtException);
worker.setWALReader(
startNewWALReader(worker.getName(), walGroupId, queue,
worker.getStartPosition()));
workerThreads.put(walGroupId, worker);
@@ -76,13 +76,13 @@ public class RecoveredReplicationSource extends
ReplicationSource {
}
@Override
- protected ReplicationSourceWALReader startNewWALReader(String threadName,
- String walGroupId, PriorityBlockingQueue queue, long
startPosition) {
-ReplicationSourceWALReader walReader = new
RecoveredReplicationSourceWALReader(fs,
-conf, queue, startPosition, walEntryFilter, this);
-Threads.setDaemonThreadRunning(walReader, threadName
-+ ".replicationSource.replicationWALReaderThread." + walGroupId + ","
+ queueId,
- getUncaughtExceptionHandler());
+ protected ReplicationSourceWALReader startNewWALReader(String threadName,
String walGroupId,
+ PriorityBlockingQueue queue, long startPosition) {
+ReplicationSourceWALReader walReader =
+ new RecoveredReplicationSourceWALReader(fs, conf, queue, startPosition,
walEntryFilter, this);
+Threads.setDaemonThreadRunning(walReader,
+ threadName + ".replicationSource.replicationWALReaderThread." +
walGroupId + "," + queueId,
+ this::uncaughtException);
return walReader;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/388fb4bd/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0092251..09b6cc1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -75,7 +75,7 @@ import
org.apache.hbase.thirdparty.com.google.common.collect.Lists;
*
*/
@InterfaceAudience.Private
-public class ReplicationSource extends Thread implements
ReplicationSourceInterface {
+public class ReplicationSource implements ReplicationSourceInterface {
private static final Logger LOG =
LoggerFactory.getLogger(ReplicationSource.class);
// Queues of logs to process, entry in format of walGroupId->queue,
@@ -114,10 +114,8 @@ public class ReplicationSource extends Thread implements
ReplicationSourceInterf
private MetricsSource metrics;
// WARN threshold for the number of queued logs, defaults to 2
private int logQueueWarnThreshold;
- // whether the replication endpoint has been initialized
- private volatile boolean endpointInitialized = false;
// ReplicationEndpoint which will handle the actual replication
- private ReplicationEndpoint replicationEndpoint;
+ private volatile ReplicationEndpoint replicationEndpoint;
// A filter (or a
[24/50] [abbrv] hbase git commit: HBASE-19592 Add UTs to test retry on update zk failure
HBASE-19592 Add UTs to test retry on update zk failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b803287
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b803287
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b803287
Branch: refs/heads/HBASE-19397-branch-2
Commit: 2b80328743a0b4fb5eb131a6a0faa7f4b1d9da4e
Parents: ff40fc5
Author: zhangduo
Authored: Tue Dec 26 20:39:00 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../replication/ReplicationPeerManager.java | 5 +-
.../TestReplicationProcedureRetry.java | 200 +++
2 files changed, 202 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b803287/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index b78cbce..f4ccce8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Used to add/remove a replication peer.
*/
@InterfaceAudience.Private
-public final class ReplicationPeerManager {
+public class ReplicationPeerManager {
private final ReplicationPeerStorage peerStorage;
@@ -61,8 +61,7 @@ public final class ReplicationPeerManager {
private final ConcurrentMap peers;
- private ReplicationPeerManager(ReplicationPeerStorage peerStorage,
- ReplicationQueueStorage queueStorage,
+ ReplicationPeerManager(ReplicationPeerStorage peerStorage,
ReplicationQueueStorage queueStorage,
ConcurrentMap peers) {
this.peerStorage = peerStorage;
this.queueStorage = queueStorage;
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b803287/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
new file mode 100644
index 000..ab35b46
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+
+/**
+ * All the modification method will fail once in the test and should finally
succeed.
+ */
+@Category({ Replica
[38/50] [abbrv] hbase git commit: HBASE-19634 Add permission check for executeProcedures in AccessController
HBASE-19634 Add permission check for executeProcedures in AccessController
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0572de37
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0572de37
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0572de37
Branch: refs/heads/HBASE-19397-branch-2
Commit: 0572de37053963f6c5ab160d8121ee8ad0f76344
Parents: 84514c0
Author: zhangduo
Authored: Thu Jan 4 16:18:21 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../hbase/coprocessor/RegionServerObserver.java | 14 ++
.../hbase/regionserver/RSRpcServices.java | 52 +++-
.../RegionServerCoprocessorHost.java| 18 +++
.../hbase/security/access/AccessController.java | 30 ++-
.../hadoop/hbase/TestJMXConnectorServer.java| 7 +++
.../security/access/TestAccessController.java | 18 +--
6 files changed, 100 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0572de37/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
index c1af3fb..5b751df 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
@@ -126,4 +126,18 @@ public interface RegionServerObserver {
default void postClearCompactionQueues(
final ObserverContext ctx)
throws IOException {}
+
+ /**
+ * This will be called before executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
preExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
+
+ /**
+ * This will be called after executing procedures
+ * @param ctx the environment to interact with the framework and region
server.
+ */
+ default void
postExecuteProcedures(ObserverContext ctx)
+ throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0572de37/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 2cdfc48..4dd826f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -3508,36 +3508,40 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
}
@Override
+ @QosPriority(priority = HConstants.ADMIN_QOS)
public ExecuteProceduresResponse executeProcedures(RpcController controller,
ExecuteProceduresRequest request) throws ServiceException {
-if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req : request.getOpenRegionList()) {
-openRegion(controller, req);
+try {
+ checkOpen();
+ regionServer.getRegionServerCoprocessorHost().preExecuteProcedures();
+ if (request.getOpenRegionCount() > 0) {
+for (OpenRegionRequest req : request.getOpenRegionList()) {
+ openRegion(controller, req);
+}
}
-}
-if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req : request.getCloseRegionList()) {
-closeRegion(controller, req);
+ if (request.getCloseRegionCount() > 0) {
+for (CloseRegionRequest req : request.getCloseRegionList()) {
+ closeRegion(controller, req);
+}
}
-}
-if (request.getProcCount() > 0) {
- for (RemoteProcedureRequest req : request.getProcList()) {
-RSProcedureCallable callable;
-try {
- callable =
-
Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
-} catch (Exception e) {
- // here we just ignore the error as this should not happen and we do
not provide a general
- // way to report errors for all types of remote procedure. The
procedure will hang at
- // master side but after you solve the problem and restart master it
will be executed
- // again and pass.
- LOG.warn("create procedure of type " + req.getProcClass() + "
failed, give up", e);
- continue;
+ if (request.getProcCount() > 0) {
+for (RemoteProcedureRequest req : request.getProcList()) {
+
[25/50] [abbrv] hbase git commit: HBASE-19564 Procedure id is missing in the response of peer related operations
HBASE-19564 Procedure id is missing in the response of peer related operations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/601818c6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/601818c6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/601818c6
Branch: refs/heads/HBASE-19397-branch-2
Commit: 601818c630d95e5f5849d4cc7459ffb5a3f1ade3
Parents: f507539
Author: zhangduo
Authored: Wed Dec 20 20:57:37 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../hadoop/hbase/master/MasterRpcServices.java | 24 ++--
.../master/replication/ModifyPeerProcedure.java | 4 +---
2 files changed, 13 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/601818c6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 7bd355a..6ce0d39 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1868,10 +1868,10 @@ public class MasterRpcServices extends RSRpcServices
public AddReplicationPeerResponse addReplicationPeer(RpcController
controller,
AddReplicationPeerRequest request) throws ServiceException {
try {
- master.addReplicationPeer(request.getPeerId(),
-ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
request.getPeerState()
-.getState().equals(ReplicationState.State.ENABLED));
- return AddReplicationPeerResponse.newBuilder().build();
+ long procId = master.addReplicationPeer(request.getPeerId(),
+ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
+
request.getPeerState().getState().equals(ReplicationState.State.ENABLED));
+ return AddReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1881,8 +1881,8 @@ public class MasterRpcServices extends RSRpcServices
public RemoveReplicationPeerResponse removeReplicationPeer(RpcController
controller,
RemoveReplicationPeerRequest request) throws ServiceException {
try {
- master.removeReplicationPeer(request.getPeerId());
- return RemoveReplicationPeerResponse.newBuilder().build();
+ long procId = master.removeReplicationPeer(request.getPeerId());
+ return
RemoveReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1892,8 +1892,8 @@ public class MasterRpcServices extends RSRpcServices
public EnableReplicationPeerResponse enableReplicationPeer(RpcController
controller,
EnableReplicationPeerRequest request) throws ServiceException {
try {
- master.enableReplicationPeer(request.getPeerId());
- return EnableReplicationPeerResponse.newBuilder().build();
+ long procId = master.enableReplicationPeer(request.getPeerId());
+ return
EnableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1903,8 +1903,8 @@ public class MasterRpcServices extends RSRpcServices
public DisableReplicationPeerResponse disableReplicationPeer(RpcController
controller,
DisableReplicationPeerRequest request) throws ServiceException {
try {
- master.disableReplicationPeer(request.getPeerId());
- return DisableReplicationPeerResponse.newBuilder().build();
+ long procId = master.disableReplicationPeer(request.getPeerId());
+ return
DisableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1930,9 +1930,9 @@ public class MasterRpcServices extends RSRpcServices
public UpdateReplicationPeerConfigResponse
updateReplicationPeerConfig(RpcController controller,
UpdateReplicationPeerConfigRequest request) throws ServiceException {
try {
- master.updateReplicationPeerConfig(request.getPeerId(),
+ long procId = master.updateReplicationPeerConfig(request.getPeerId(),
ReplicationPeerConfigUtil.convert(request.getPeerConfig()));
- return UpdateReplicationPeerConfigResponse.newBuilder().build();
+ return
UpdateReplicationPeerConfigResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
thro
[07/50] [abbrv] hbase git commit: HBASE-20093 (addendum) remove unused import of ServerLoad
HBASE-20093 (addendum) remove unused import of ServerLoad
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2bf2ad7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2bf2ad7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2bf2ad7
Branch: refs/heads/HBASE-19397-branch-2
Commit: a2bf2ad71eb58bd6708f62c6defbe82d17fb1166
Parents: 03f9cb8
Author: Chia-Ping Tsai
Authored: Thu Mar 1 23:43:40 2018 +0800
Committer: Chia-Ping Tsai
Committed: Thu Mar 1 23:44:29 2018 +0800
--
.../org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon| 1 -
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 3 +--
2 files changed, 1 insertion(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2bf2ad7/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 12e0a69b..3091e18 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -37,7 +37,6 @@ org.apache.hadoop.hbase.HBaseConfiguration;
org.apache.hadoop.hbase.HConstants;
org.apache.hadoop.hbase.HTableDescriptor;
org.apache.hadoop.hbase.NamespaceDescriptor;
-org.apache.hadoop.hbase.ServerLoad;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.TableName;
org.apache.hadoop.hbase.client.Admin;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2bf2ad7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 19b2101..b928e52 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
-import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -1038,7 +1037,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
// The isServerOnline check is opportunistic, correctness is handled
inside
if (!this.serverManager.isServerOnline(sn) &&
- serverManager.checkAndRecordNewServer(sn, new
ServerLoad(ServerMetricsBuilder.of(sn {
+ serverManager.checkAndRecordNewServer(sn,
ServerMetricsBuilder.of(sn))) {
LOG.info("Registered server found up in zk but who has not yet
reported in: " + sn);
}
}
[26/50] [abbrv] hbase git commit: HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure
HBASE-19536 Client side changes for moving peer modification from zk watcher to
procedure
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f5075395
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f5075395
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f5075395
Branch: refs/heads/HBASE-19397-branch-2
Commit: f5075395798530eb112fca110e789d1b50c591a0
Parents: ee7d9b3
Author: Guanghao Zhang
Authored: Tue Dec 19 15:50:57 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../org/apache/hadoop/hbase/client/Admin.java | 87 ++-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 149 ++-
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 82 +-
3 files changed, 238 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f5075395/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 40dac2f..b8546fa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2473,7 +2473,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @throws IOException if a remote or network exception occurs
*/
default void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig)
@@ -2484,7 +2484,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @param enabled peer state, true if ENABLED and false if DISABLED
* @throws IOException if a remote or network exception occurs
*/
@@ -2492,6 +2492,37 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ default Future addReplicationPeerAsync(String peerId,
ReplicationPeerConfig peerConfig)
+ throws IOException {
+return addReplicationPeerAsync(peerId, peerConfig, true);
+ }
+
+ /**
+ * Add a new replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @param enabled peer state, true if ENABLED and false if DISABLED
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future addReplicationPeerAsync(String peerId, ReplicationPeerConfig
peerConfig,
+ boolean enabled) throws IOException;
+
+ /**
* Remove a peer and stop the replication.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2499,6 +2530,18 @@ public interface Admin extends Abortable, Closeable {
void removeReplicationPeer(String peerId) throws IOException;
/**
+ * Remove a replication peer but does not block and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * ExecutionException if there was an error while executing the operation or
TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to
complete.
+ * @param peerId a short
[42/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
HBASE-19622 Reimplement ReplicationPeers with the new replication storage
interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/505f2517
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/505f2517
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/505f2517
Branch: refs/heads/HBASE-19397-branch-2
Commit: 505f251772bc20cbf0b3abde1448c6d9befa8d41
Parents: 84d5e48
Author: huzheng
Authored: Tue Dec 26 16:46:10 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../replication/ReplicationPeerConfigUtil.java | 10 +-
.../replication/VerifyReplication.java | 9 +-
.../hbase/replication/ReplicationFactory.java | 10 +-
.../hbase/replication/ReplicationPeerImpl.java | 60 +-
.../replication/ReplicationPeerStorage.java | 3 +-
.../hbase/replication/ReplicationPeers.java | 238
.../replication/ReplicationPeersZKImpl.java | 552 ---
.../replication/ZKReplicationPeerStorage.java | 12 +-
.../replication/ZKReplicationStorageBase.java | 3 +-
.../replication/TestReplicationStateBasic.java | 125 ++---
.../replication/TestReplicationStateZKImpl.java | 2 +-
.../TestZKReplicationPeerStorage.java | 12 +-
.../cleaner/ReplicationZKNodeCleaner.java | 57 +-
.../replication/ReplicationPeerManager.java | 6 +-
.../regionserver/DumpReplicationQueues.java | 2 +-
.../regionserver/PeerProcedureHandlerImpl.java | 49 +-
.../replication/regionserver/Replication.java | 2 +-
.../regionserver/ReplicationSource.java | 7 +-
.../regionserver/ReplicationSourceManager.java | 44 +-
.../cleaner/TestReplicationHFileCleaner.java| 7 +-
.../replication/TestMultiSlaveReplication.java | 2 -
.../TestReplicationTrackerZKImpl.java | 26 +-
.../TestReplicationSourceManager.java | 17 +-
.../hadoop/hbase/HBaseZKTestingUtility.java | 3 +-
24 files changed, 307 insertions(+), 951 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/505f2517/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 022bf64..a234a9b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -247,22 +247,22 @@ public final class ReplicationPeerConfigUtil {
public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
throws DeserializationException {
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
- int pblen = ProtobufUtil.lengthOfPBMagic();
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
ReplicationProtos.ReplicationPeer.Builder builder =
ReplicationProtos.ReplicationPeer.newBuilder();
ReplicationProtos.ReplicationPeer peer;
try {
-ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
peer = builder.build();
} catch (IOException e) {
throw new DeserializationException(e);
}
return convert(peer);
} else {
- if (bytes.length > 0) {
-return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
+ if (bytes == null || bytes.length <= 0) {
+throw new DeserializationException("Bytes to deserialize should not be
empty.");
}
- return ReplicationPeerConfig.newBuilder().setClusterKey("").build();
+ return
ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/505f2517/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 09d4b4b..f0070f0 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -339,15 +339,10 @@ public class VerifyReplication extends Configured
implements Tool {
@Override public boolean
[47/50] [abbrv] hbase git commit: HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed
HBASE-19783 Change replication peer cluster key/endpoint from a not-null value
to null is not allowed
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/557c4449
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/557c4449
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/557c4449
Branch: refs/heads/HBASE-19397-branch-2
Commit: 557c444993deafaf548776dbfc75a24347ae3150
Parents: 040b871
Author: Guanghao Zhang
Authored: Fri Jan 12 22:04:38 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../replication/ReplicationPeerManager.java | 28 +---
1 file changed, 19 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/557c4449/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 696b2d7..19fc7f4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -132,20 +132,19 @@ public class ReplicationPeerManager {
checkPeerConfig(peerConfig);
ReplicationPeerDescription desc = checkPeerExists(peerId);
ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
-if (!StringUtils.isBlank(peerConfig.getClusterKey()) &&
- !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) {
+if (!isStringEquals(peerConfig.getClusterKey(),
oldPeerConfig.getClusterKey())) {
throw new DoNotRetryIOException(
"Changing the cluster key on an existing peer is not allowed.
Existing key '" +
-oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
-peerConfig.getClusterKey() + "'");
+ oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does
not match new key '" +
+ peerConfig.getClusterKey() + "'");
}
-if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) &&
-
!peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl()))
{
+if (!isStringEquals(peerConfig.getReplicationEndpointImpl(),
+ oldPeerConfig.getReplicationEndpointImpl())) {
throw new DoNotRetryIOException("Changing the replication endpoint
implementation class " +
-"on an existing peer is not allowed. Existing class '" +
-oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
-" does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
+ "on an existing peer is not allowed. Existing class '" +
+ oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
+ " does not match new class '" +
peerConfig.getReplicationEndpointImpl() + "'");
}
}
@@ -341,4 +340,15 @@ public class ReplicationPeerManager {
return new ReplicationPeerManager(peerStorage,
ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
}
+
+ /**
+ * For replication peer cluster key or endpoint class, null and empty string
is same. So here
+ * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly.
+ */
+ private boolean isStringEquals(String s1, String s2) {
+if (StringUtils.isBlank(s1)) {
+ return StringUtils.isBlank(s2);
+}
+return s1.equals(s2);
+ }
}
[22/50] [abbrv] hbase git commit: HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bfef7d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bfef7d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bfef7d9
Branch: refs/heads/HBASE-19397-branch-2
Commit: 5bfef7d9516c924754329c0a97d76aee22da8da8
Parents: 7674225
Author: Guanghao Zhang
Authored: Tue Dec 26 11:39:34 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../replication/VerifyReplication.java | 5 -
.../hbase/replication/ReplicationPeer.java | 42 ++--
.../hbase/replication/ReplicationPeerImpl.java | 169 ++
.../replication/ReplicationPeerZKImpl.java | 233 ---
.../hbase/replication/ReplicationPeers.java | 4 +-
.../replication/ReplicationPeersZKImpl.java | 23 +-
.../replication/TestReplicationStateBasic.java | 7 +-
.../regionserver/PeerProcedureHandlerImpl.java | 29 +--
8 files changed, 216 insertions(+), 296 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bfef7d9/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 9065f4e..09d4b4b 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
@@ -333,7 +332,6 @@ public class VerifyReplication extends Configured
implements Tool {
private static Pair
getPeerQuorumConfig(
final Configuration conf, String peerId) throws IOException {
ZKWatcher localZKW = null;
-ReplicationPeerZKImpl peer = null;
try {
localZKW = new ZKWatcher(conf, "VerifyReplication",
new Abortable() {
@@ -354,9 +352,6 @@ public class VerifyReplication extends Configured
implements Tool {
throw new IOException(
"An error occurred while trying to connect to the remove peer
cluster", e);
} finally {
- if (peer != null) {
-peer.close();
- }
if (localZKW != null) {
localZKW.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bfef7d9/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index b66d76d..4846018 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* ReplicationPeer manages enabled / disabled state for the peer.
*/
@@ -49,65 +48,52 @@ public interface ReplicationPeer {
String getId();
/**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig();
-
- /**
- * Get the peer config object. if loadFromBackingStore is true, it will load
from backing store
- * directly and update its load peer config. otherwise, just return the
local cached peer config.
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
- throws ReplicationException;
-
- /**
* Returns the state of the peer by reading local cache.
* @return the enabled state
*/
PeerState getPeerState();
/**
- * Returns the state of peer, if loadFromBackingStore is true, it will load
from backing store
- * directly and update its local peer state. otherwise, just return th
[31/50] [abbrv] hbase git commit: HBASE-19719 Fix checkstyle issues
HBASE-19719 Fix checkstyle issues
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ee4e182
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ee4e182
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ee4e182
Branch: refs/heads/HBASE-19397-branch-2
Commit: 5ee4e182594b1c77ea0732d1f5e1bdacb3dec29b
Parents: 855ea3b
Author: zhangduo
Authored: Sat Jan 6 08:30:55 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../hbase/replication/ReplicationStorageFactory.java | 2 +-
.../master/assignment/RegionTransitionProcedure.java | 4 ++--
.../hbase/master/procedure/RSProcedureDispatcher.java | 13 ++---
.../master/ReplicationPeerConfigUpgrader.java | 8
4 files changed, 13 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ee4e182/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 60d0749..462cfed 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -27,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* For now we only have zk based implementation.
*/
@InterfaceAudience.Private
-public class ReplicationStorageFactory {
+public final class ReplicationStorageFactory {
private ReplicationStorageFactory() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ee4e182/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index a0e58f3..7ce7454 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -36,11 +36,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
/**
* Base class for the Assign and Unassign Procedure.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/5ee4e182/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 57a4535..6c78914 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
@@ -36,6 +35,12 @@ import org.apache.hadoop.ipc.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
@@ -47,12 +52,6 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR
import
org.apache.hadoop.hbase.shaded.protobuf
[04/50] [abbrv] hbase git commit: HBASE-19863 java.lang.IllegalStateException: isDelete failed when SingleColumnValueFilter is used
HBASE-19863 java.lang.IllegalStateException: isDelete failed when
SingleColumnValueFilter is used
Signed-off-by: Chia-Ping Tsai
Signed-off-by: ramkrish86
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/313464f0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/313464f0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/313464f0
Branch: refs/heads/HBASE-19397-branch-2
Commit: 313464f007b724bf83b1e19b9e90f8e619e781a4
Parents: 2f3632a
Author: Sergey Soldatov
Authored: Tue Feb 13 22:08:11 2018 -0800
Committer: Josh Elser
Committed: Wed Feb 28 14:06:17 2018 -0500
--
.../hadoop/hbase/regionserver/StoreScanner.java | 6 +
.../hadoop/hbase/HBaseTestingUtility.java | 41 +++--
.../hbase/regionserver/TestIsDeleteFailure.java | 160 +++
3 files changed, 195 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/313464f0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index 1624810..9f6a015 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -816,6 +816,12 @@ public class StoreScanner extends
NonReversedNonLazyKeyValueScanner
return false;
}
} while ((nextCell = this.heap.peek()) != null &&
CellUtil.matchingRowColumn(cell, nextCell));
+// We need this check because it may happen that the new scanner that we
get
+// during heap.next() is requiring reseek due of fake KV previously
generated for
+// ROWCOL bloom filter optimization. See HBASE-19863 for more details
+if (nextCell != null && matcher.compareKeyForNextColumn(nextCell, cell) <
0) {
+ return false;
+}
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/313464f0/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 2bdfd2d..b48abc6 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1378,23 +1378,40 @@ public class HBaseTestingUtility extends
HBaseZKTestingUtility {
/**
* Create a table.
- * @param htd
- * @param families
- * @param splitKeys
+ * @param htd table descriptor
+ * @param families array of column families
+ * @param splitKeys array of split keys
* @param c Configuration to use
* @return A Table instance for the created table.
- * @throws IOException
+ * @throws IOException if getAdmin or createTable fails
*/
public Table createTable(TableDescriptor htd, byte[][] families, byte[][]
splitKeys,
Configuration c) throws IOException {
+// Disable blooms (they are on by default as of 0.95) but we disable them
here because
+// tests have hard coded counts of what to expect in block cache, etc.,
and blooms being
+// on is interfering.
+return createTable(htd, families, splitKeys, BloomType.NONE,
HConstants.DEFAULT_BLOCKSIZE, c);
+ }
+
+ /**
+ * Create a table.
+ * @param htd table descriptor
+ * @param families array of column families
+ * @param splitKeys array of split keys
+ * @param type Bloom type
+ * @param blockSize block size
+ * @param c Configuration to use
+ * @return A Table instance for the created table.
+ * @throws IOException if getAdmin or createTable fails
+ */
+
+ public Table createTable(TableDescriptor htd, byte[][] families, byte[][]
splitKeys,
+ BloomType type, int blockSize, Configuration c) throws IOException {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd);
for (byte[] family : families) {
- // Disable blooms (they are on by default as of 0.95) but we disable
them here because
- // tests have hard coded counts of what to expect in block cache, etc.,
and blooms being
- // on is interfering.
- builder.addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family)
- .setBloomFilterType(BloomType.NONE)
- .build());
+ builder.addColumnFamily(
+
ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(type)
+ .setBlocksize(blockSize).bu
[08/50] [abbrv] hbase git commit: HBASE-20104 Fix infinite loop of RIT when creating table on a rsgroup that has no online servers
HBASE-20104 Fix infinite loop of RIT when creating table on a rsgroup that has
no online servers
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a08ade9f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a08ade9f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a08ade9f
Branch: refs/heads/HBASE-19397-branch-2
Commit: a08ade9f1aa733b41b1f6fb0b5f7cd6643c4b8a9
Parents: a2bf2ad
Author: haxiaolin
Authored: Wed Feb 28 19:15:57 2018 +0800
Committer: tedyu
Committed: Thu Mar 1 08:34:58 2018 -0800
--
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 34 +++-
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 43
2 files changed, 76 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a08ade9f/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index ce74bb4..0006d68 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.rsgroup;
-
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
@@ -32,6 +31,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
@@ -353,6 +353,27 @@ public class RSGroupAdminEndpoint implements
MasterCoprocessor, MasterObserver {
}
}
+ boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException {
+String groupName =
+
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
+.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
+if (groupName == null) {
+ groupName = RSGroupInfo.DEFAULT_GROUP;
+}
+RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
+if (rsGroupInfo == null) {
+ throw new ConstraintException(
+ "Default RSGroup (" + groupName + ") for this table's " + "namespace
does not exist.");
+}
+
+for (ServerName onlineServer :
master.getServerManager().createDestinationServersList()) {
+ if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) {
+return true;
+ }
+}
+return false;
+ }
+
void assignTableToGroup(TableDescriptor desc) throws IOException {
String groupName =
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
@@ -375,6 +396,17 @@ public class RSGroupAdminEndpoint implements
MasterCoprocessor, MasterObserver {
// MasterObserver overrides
/
+ @Override
+ public void preCreateTableAction(
+ final ObserverContext ctx,
+ final TableDescriptor desc,
+ final RegionInfo[] regions) throws IOException {
+if (!desc.getTableName().isSystemTable() &&
!rsgroupHasServersOnline(desc)) {
+ throw new HBaseIOException("No online servers in the rsgroup, which
table " +
+ desc.getTableName().getNameAsString() + " belongs to");
+}
+ }
+
// Assign table to default RSGroup.
@Override
public void postCreateTable(ObserverContext
ctx,
http://git-wip-us.apache.org/repos/asf/hbase/blob/a08ade9f/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index fdc29b3..9422bf8 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -997,4 +997,47 @@ public abstract class TestRSGroupsBase {
assertFalse(newGroupServers.contains(targetServer.getAddress()));
assertEquals(2, newGroupServers.size());
}
+
+ @Test
+ public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
+LOG.info("testCreateWhenRsgroupNoOnlineServers");
+
+// set rsgroup has no online servers and test create table
+final RSGroupInfo appIn
[17/50] [abbrv] hbase git commit: HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure
HBASE-19525 RS side changes for moving peer modification from zk watcher to
procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5d0e061a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5d0e061a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5d0e061a
Branch: refs/heads/HBASE-19397-branch-2
Commit: 5d0e061a1e1cceff9342b99fa7311afc5e4f3d85
Parents: 4da84ed
Author: huzheng
Authored: Wed Dec 20 10:47:18 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../hadoop/hbase/protobuf/ProtobufUtil.java | 11 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 13 +-
.../hbase/replication/ReplicationListener.java | 14 --
.../hbase/replication/ReplicationPeer.java | 28 ++-
.../replication/ReplicationPeerZKImpl.java | 186 ---
.../replication/ReplicationPeersZKImpl.java | 19 +-
.../replication/ReplicationTrackerZKImpl.java | 73 +-
.../regionserver/ReplicationSourceService.java | 6 +
.../handler/RSProcedureHandler.java | 3 +
.../replication/BaseReplicationEndpoint.java| 2 +-
.../regionserver/PeerProcedureHandler.java | 38
.../regionserver/PeerProcedureHandlerImpl.java | 81 +++
.../regionserver/RefreshPeerCallable.java | 39 +++-
.../replication/regionserver/Replication.java | 9 +
.../regionserver/ReplicationSource.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 37 ++-
.../TestReplicationAdminUsingProcedure.java | 226 +++
.../replication/DummyModifyPeerProcedure.java | 48
.../TestDummyModifyPeerProcedure.java | 80 ---
.../TestReplicationTrackerZKImpl.java | 61 -
.../TestReplicationSourceManager.java | 32 ++-
21 files changed, 532 insertions(+), 482 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5d0e061a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 3033da7..f11fcf6 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -191,7 +193,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMagic#PB_MAGIC}.length.
*/
public static byte [] prependPBMagic(final byte [] bytes) {
-return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
+return Bytes.add(PB_MAGIC, bytes);
}
/**
@@ -216,10 +218,11 @@ public final class ProtobufUtil {
* @param bytes bytes to check
* @throws DeserializationException if we are missing the pb magic prefix
*/
- public static void expectPBMagicPrefix(final byte [] bytes) throws
DeserializationException {
+ public static void expectPBMagicPrefix(final byte[] bytes) throws
DeserializationException {
if (!isPBMagicPrefix(bytes)) {
- throw new DeserializationException("Missing pb magic " +
- Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
+ String bytesPrefix = bytes == null ? "null" :
Bytes.toStringBinary(bytes, 0, PB_MAGIC.length);
+ throw new DeserializationException(
+ "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix, bytes: "
+ bytesPrefix);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5d0e061a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 4472846..520a4cd 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.shaded.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -277,7 +279,7 @@ public final class ProtobufUtil {
* byte array that is bytes.length plus {@link
ProtobufMagi
[45/50] [abbrv] hbase git commit: HBASE-19936 Introduce a new base class for replication peer procedure
HBASE-19936 Introduce a new base class for replication peer procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1b2ad69
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1b2ad69
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1b2ad69
Branch: refs/heads/HBASE-19397-branch-2
Commit: c1b2ad69d5a72c141478411028a04c7a0ef06cc3
Parents: f6727de
Author: zhangduo
Authored: Mon Feb 5 16:14:25 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 2 +-
.../replication/AbstractPeerProcedure.java | 97
.../master/replication/ModifyPeerProcedure.java | 67 +-
3 files changed, 102 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b2ad69/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index a9521ad..9377988 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -401,7 +401,7 @@ message RefreshPeerParameter {
required ServerName target_server = 3;
}
-message ModifyPeerStateData {
+message PeerProcedureStateData {
required string peer_id = 1;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b2ad69/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
new file mode 100644
index 000..0ad8a63
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerProcedureStateData;
+
+/**
+ * The base class for all replication peer related procedure.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractPeerProcedure
+extends StateMachineProcedure implements
PeerProcedureInterface {
+
+ protected String peerId;
+
+ private volatile boolean locked;
+
+ // used to keep compatible with old client where we can only returns after
updateStorage.
+ protected ProcedurePrepareLatch latch;
+
+ protected AbstractPeerProcedure() {
+ }
+
+ protected AbstractPeerProcedure(String peerId) {
+this.peerId = peerId;
+this.latch = ProcedurePrepareLatch.createLatch(2, 0);
+ }
+
+ public ProcedurePrepareLatch getLatch() {
+return latch;
+ }
+
+ @Override
+ public String getPeerId() {
+return peerId;
+ }
+
+ @Override
+ protected LockState acquireLock(MasterProcedureEnv env) {
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
+ }
+
+ @Override
+ protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
+env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
+ }
+
+ @Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protected
[14/50] [abbrv] hbase git commit: HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes
HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer
Procedure classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4da84ed6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4da84ed6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4da84ed6
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4da84ed63c397860e56ffb4ea97fa75c32513112
Parents: 69432ea
Author: zhangduo
Authored: Thu Dec 21 21:59:46 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../hadoop/hbase/master/replication/AddPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/DisablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/EnablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/ModifyPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RemovePeerProcedure.java | 6 +++---
.../hbase/master/replication/UpdatePeerConfigProcedure.java| 6 +++---
7 files changed, 21 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4da84ed6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index c3862d8..066c3e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -28,6 +26,8 @@ import
org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData;
@@ -37,7 +37,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.A
@InterfaceAudience.Private
public class AddPeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(AddPeerProcedure.class);
private ReplicationPeerConfig peerConfig;
http://git-wip-us.apache.org/repos/asf/hbase/blob/4da84ed6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 0b32db9..9a28de6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for disabling a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DisablePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(DisablePeerProcedure.class);
public DisablePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4da84ed6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
--
diff --git
a/hbase-server/src/main
[02/50] [abbrv] hbase git commit: HBASE-20097 Merge TableDescriptors#getAll and TableDescriptors#getAllDescriptors into one
HBASE-20097 Merge TableDescriptors#getAll and
TableDescriptors#getAllDescriptors into one
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f71c00c0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f71c00c0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f71c00c0
Branch: refs/heads/HBASE-19397-branch-2
Commit: f71c00c02e7a27d9cd6ff1de8c320b818ebb6221
Parents: e50e6f7
Author: Chia-Ping Tsai
Authored: Tue Feb 27 16:34:14 2018 +0800
Committer: Chia-Ping Tsai
Committed: Wed Feb 28 15:21:31 2018 +0800
--
.../org/apache/hadoop/hbase/TableDescriptors.java | 9 -
.../hadoop/hbase/master/TableStateManager.java | 2 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 16 +---
.../hbase/master/assignment/MockMasterServices.java | 5 -
4 files changed, 2 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f71c00c0/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index ecdfc82..5787f66 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -55,15 +55,6 @@ public interface TableDescriptors {
throws IOException;
/**
- * Get Map of all TableDescriptors. Populates the descriptor cache as a
- * side effect.
- * @return Map of all descriptors.
- * @throws IOException
- */
- Map getAllDescriptors()
- throws IOException;
-
- /**
* Add or update descriptor
* @param htd Descriptor to set into TableDescriptors
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/f71c00c0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
index 0d89eef..affb684 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
@@ -223,7 +223,7 @@ public class TableStateManager {
private void fixTableStates(TableDescriptors tableDescriptors, Connection
connection)
throws IOException {
-final Map allDescriptors =
tableDescriptors.getAllDescriptors();
+final Map allDescriptors =
tableDescriptors.getAll();
final Map states = new HashMap<>();
// NOTE: Ful hbase:meta table scan!
MetaTableAccessor.fullScanTables(connection, new
MetaTableAccessor.Visitor() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f71c00c0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index c72b9e0..b4b0be0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -248,7 +248,7 @@ public class FSTableDescriptors implements TableDescriptors
{
* Returns a map from table name to table descriptor for all tables.
*/
@Override
- public Map getAllDescriptors()
+ public Map getAll()
throws IOException {
Map tds = new TreeMap<>();
@@ -282,20 +282,6 @@ public class FSTableDescriptors implements
TableDescriptors {
}
/**
- * Returns a map from table name to table descriptor for all tables.
- */
- @Override
- public Map getAll() throws IOException {
-Map htds = new TreeMap<>();
-Map allDescriptors = getAllDescriptors();
-for (Map.Entry entry : allDescriptors
-.entrySet()) {
- htds.put(entry.getKey(), entry.getValue());
-}
-return htds;
- }
-
- /**
* Find descriptors by namespace.
* @see #get(org.apache.hadoop.hbase.TableName)
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/f71c00c0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterSe
[39/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbc81ff5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index b8d80d2..5b7bab8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -33,17 +33,20 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -56,7 +59,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -91,7 +94,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
private final List sources;
// List of all the sources we got from died RSs
private final List oldsources;
- private final ReplicationQueues replicationQueues;
+ private final ReplicationQueueStorage queueStorage;
private final ReplicationTracker replicationTracker;
private final ReplicationPeers replicationPeers;
// UUID for this cluster
@@ -124,7 +127,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
/**
* Creates a replication manager and sets the watch on all the other
registered region servers
- * @param replicationQueues the interface for manipulating replication queues
+ * @param queueStorage the interface for manipulating replication queues
* @param replicationPeers
* @param replicationTracker
* @param conf the configuration to use
@@ -134,14 +137,14 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param oldLogDir the directory where old logs are archived
* @param clusterId
*/
- public ReplicationSourceManager(ReplicationQueues replicationQueues,
+ public ReplicationSourceManager(ReplicationQueueStorage queueStorage,
ReplicationPeers replicationPeers, ReplicationTracker
replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID
clusterId,
WALFileLengthProvider walFileLengthProvider) throws IOException {
//CopyOnWriteArrayList is thread-safe.
//Generally, reading is more than modifying.
this.sources = new CopyOnWriteArrayList<>();
-this.replicationQueues = replicationQueues;
+this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.replicationTracker = replicationTracker;
this.server = server;
@@ -174,6 +177,19 @@ public class ReplicationSourceManager implements
ReplicationListener {
HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
}
+ @FunctionalInterface
+ private interface ReplicationQueueOperation {
+void exec() throws ReplicationException;
+ }
+
+ private void abortWhenFail(ReplicationQueueOperation op) {
+try {
+ op.exec();
+} catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+}
+ }
+
/**
* Provide the id of the peer and a log key and this method will figure which
* wal it belongs to and will log, for this region server, the current
@@ -185,12 +201,13 @@ public class ReplicationSourceManager implements
ReplicationListener {
* @param queueRecovered indicates if this queue comes from another region
server
* @param holdLogInZK if true then the log is retained in
[27/50] [abbrv] hbase git commit: HBASE-19544 Add UTs for testing concurrent modifications on replication peer
HBASE-19544 Add UTs for testing concurrent modifications on replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e39dbec9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e39dbec9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e39dbec9
Branch: refs/heads/HBASE-19397-branch-2
Commit: e39dbec91cf2f8cc82b4a5cfc3abbc6bb38c6fcd
Parents: 45ee0d5
Author: Guanghao Zhang
Authored: Tue Jan 2 17:07:41 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../replication/TestReplicationAdmin.java | 69
1 file changed, 69 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e39dbec9/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 772a9d6..a753d23 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -58,6 +59,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Unit testing of ReplicationAdmin
@@ -69,6 +72,8 @@ public class TestReplicationAdmin {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestReplicationAdmin.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(TestReplicationAdmin.class);
+
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
@@ -118,6 +123,70 @@ public class TestReplicationAdmin {
}
@Test
+ public void testConcurrentPeerOperations() throws Exception {
+int threadNum = 5;
+AtomicLong successCount = new AtomicLong(0);
+
+// Test concurrent add peer operation
+Thread[] addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent remove peer operation
+successCount.set(0);
+Thread[] removePeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ removePeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.removeReplicationPeer(ID_ONE);
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when remove replication peer", e);
+}
+ });
+ removePeers[i].start();
+}
+for (Thread removePeer : removePeers) {
+ removePeer.join();
+}
+assertEquals(1, successCount.get());
+
+// Test concurrent add peer operation again
+successCount.set(0);
+addPeers = new Thread[threadNum];
+for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+} catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+}
+ });
+ addPeers[i].start();
+}
+for (Thread addPeer : addPeers) {
+ addPeer.join();
+}
+assertEquals(1, successCount.get());
+ }
+
+ @Test
public void testAddInvalidPeer() {
ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
builder.setClusterKey(KEY_ONE);
[43/50] [abbrv] hbase git commit: HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer
HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f9332ad Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f9332ad Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f9332ad Branch: refs/heads/HBASE-19397-branch-2 Commit: 6f9332ade5ad409c19d3bafcd3af9d4dfad84352 Parents: 505f251 Author: zhangduo Authored: Tue Jan 2 09:57:23 2018 +0800 Committer: zhangduo Committed: Fri Mar 2 14:57:02 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 2 +- .../replication/VerifyReplication.java | 34 ++ .../hbase/replication/ReplicationPeers.java | 32 +++-- .../replication/ZKReplicationQueueStorage.java | 3 +- .../replication/ZKReplicationStorageBase.java | 4 +-- .../replication/TestReplicationStateBasic.java | 10 ++ .../master/replication/AddPeerProcedure.java| 5 ++- .../replication/DisablePeerProcedure.java | 3 +- .../master/replication/EnablePeerProcedure.java | 3 +- .../master/replication/ModifyPeerProcedure.java | 34 ++ .../replication/RefreshPeerProcedure.java | 17 - .../master/replication/RemovePeerProcedure.java | 7 ++-- .../replication/ReplicationPeerManager.java | 31 +++- .../replication/UpdatePeerConfigProcedure.java | 3 +- .../RemoteProcedureResultReporter.java | 3 +- .../regionserver/RefreshPeerCallable.java | 5 +-- .../regionserver/ReplicationSourceManager.java | 38 +++- .../TestReplicationAdminUsingProcedure.java | 7 ++-- 18 files changed, 124 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6f9332ad/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index b80ee16..fdae288 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -27,8 +27,8 @@ import java.util.Set; import java.util.TreeMap; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; /** * A configuration for the replication peer cluster. http://git-wip-us.apache.org/repos/asf/hbase/blob/6f9332ad/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java -- diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index f0070f0..fe45762 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.mapreduce.replication; import java.io.IOException; import java.util.Arrays; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -45,13 +44,14 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapper; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat; import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerStorage; import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.replication.ReplicationStorageFactory; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -66,6 +66,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; impo
[41/50] [abbrv] hbase git commit: HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
http://git-wip-us.apache.org/repos/asf/hbase/blob/505f2517/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 5b7bab8..ce9882a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -160,7 +160,6 @@ public class ReplicationSourceManager implements
ReplicationListener {
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationTracker.registerListener(this);
-this.replicationPeers.getAllPeerIds();
// It's preferable to failover 1 RS at a time, but with good zk servers
// more could be processed at the same time.
int nbWorkers = conf.getInt("replication.executor.workers", 1);
@@ -260,8 +259,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
}
List otherRegionServers =
replicationTracker.getListOfRegionServers().stream()
.map(ServerName::valueOf).collect(Collectors.toList());
-LOG.info(
- "Current list of replicators: " + currentReplicators + " other RSs: " +
otherRegionServers);
+LOG.info("Current list of replicators: " + currentReplicators + " other
RSs: "
++ otherRegionServers);
// Look if there's anything to process after a restart
for (ServerName rs : currentReplicators) {
@@ -278,7 +277,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
* The returned future is for adoptAbandonedQueues task.
*/
Future init() throws IOException, ReplicationException {
-for (String id : this.replicationPeers.getConnectedPeerIds()) {
+for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
if (replicationForBulkLoadDataEnabled) {
// Check if peer exists in hfile-refs queue, if not add it. This can
happen in the case
@@ -297,8 +296,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException,
ReplicationException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(id);
-ReplicationPeer peer = replicationPeers.getConnectedPeer(id);
+ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id);
+ReplicationPeer peer = replicationPeers.getPeer(id);
ReplicationSourceInterface src = getReplicationSource(id, peerConfig,
peer);
synchronized (this.walsById) {
this.sources.add(src);
@@ -344,7 +343,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void deleteSource(String peerId, boolean closeConnection) {
abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(),
peerId));
if (closeConnection) {
- this.replicationPeers.peerDisconnected(peerId);
+ this.replicationPeers.removePeer(peerId);
}
}
@@ -437,12 +436,12 @@ public class ReplicationSourceManager implements
ReplicationListener {
// update replication queues on ZK
// synchronize on replicationPeers to avoid adding source for the
to-be-removed peer
synchronized (replicationPeers) {
- for (String id : replicationPeers.getConnectedPeerIds()) {
+ for (String id : replicationPeers.getAllPeerIds()) {
try {
this.queueStorage.addWAL(server.getServerName(), id, logName);
} catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue" +
-" when creating a new source, queueId=" + id + ", filename=" +
logName, e);
+ throw new IOException("Cannot add log to replication queue"
+ + " when creating a new source, queueId=" + id + ", filename=" +
logName, e);
}
}
}
@@ -587,7 +586,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
public void addPeer(String id) throws ReplicationException, IOException {
LOG.info("Trying to add peer, peerId: " + id);
-boolean added = this.replicationPeers.peerConnected(id);
+boolean added = this.replicationPeers.addPeer(id);
if (added) {
LOG.info("Peer " + id + " connected success, trying to start the
replication source thread.");
addSource(id);
@@ -723,16 +722,21 @@ public class ReplicationSourceManager implements
ReplicationListener {
// there is not an actual peer defined corresponding to peerId for
the failover.
ReplicationQueueInfo replicationQueueInfo = new
ReplicationQueueInfo(peerId);
[36/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
HBASE-19636 All rs should already start work with the new peer change when
replication peer procedure is finished
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d5277a27
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d5277a27
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d5277a27
Branch: refs/heads/HBASE-19397-branch-2
Commit: d5277a271d9428090b68b4e3facca065bef4e0e2
Parents: 0572de3
Author: Guanghao Zhang
Authored: Thu Jan 4 16:58:01 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../replication/ReplicationPeerConfig.java | 1 -
.../hbase/replication/ReplicationPeerImpl.java | 4 +-
.../hbase/replication/ReplicationQueueInfo.java | 23 +-
.../hbase/replication/ReplicationUtils.java | 56 ++
.../replication/TestReplicationStateZKImpl.java | 21 -
.../regionserver/ReplicationSourceService.java | 3 +-
.../regionserver/PeerProcedureHandler.java | 3 +
.../regionserver/PeerProcedureHandlerImpl.java | 50 +-
.../RecoveredReplicationSource.java | 6 +-
.../RecoveredReplicationSourceShipper.java | 8 +-
.../replication/regionserver/Replication.java | 11 +-
.../regionserver/ReplicationSource.java | 34 +-
.../regionserver/ReplicationSourceFactory.java | 4 +-
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 827 ++-
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALReader.java | 2 +-
.../replication/ReplicationSourceDummy.java | 2 +-
.../replication/TestNamespaceReplication.java | 57 +-
.../TestReplicationSourceManager.java | 5 +-
20 files changed, 622 insertions(+), 509 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d5277a27/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index fdae288..bf8d030 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d5277a27/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 3e17025..604e0bb 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ReplicationPeerImpl implements ReplicationPeer {
+
private final Configuration conf;
private final String id;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d5277a27/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
index ecd888f..cd65f9b 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ServerName;
/**
- * This class is responsible for the parsing logic for a znode representing a
queue.
+ * This class is responsible for the parsing logic for a queue id representing
a queue.
* It will extract the peerId if it's recovere
[11/50] [abbrv] hbase git commit: HBASE-19216 Implement a general framework to execute remote procedure on RS
HBASE-19216 Implement a general framework to execute remote procedure on RS
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25392608
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25392608
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25392608
Branch: refs/heads/HBASE-19397-branch-2
Commit: 2539260845757692e4a306aa0426d3169540c24d
Parents: a2de295
Author: zhangduo
Authored: Fri Dec 15 21:06:44 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:51:38 2018 +0800
--
.../hbase/procedure2/LockedResourceType.java| 4 +-
.../procedure2/RemoteProcedureDispatcher.java | 23 ++-
.../src/main/protobuf/Admin.proto | 9 +-
.../src/main/protobuf/MasterProcedure.proto | 30 +++
.../src/main/protobuf/RegionServerStatus.proto | 15 ++
.../apache/hadoop/hbase/executor/EventType.java | 26 ++-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 30 ++-
.../hadoop/hbase/master/MasterRpcServices.java | 13 ++
.../assignment/RegionTransitionProcedure.java | 18 +-
.../procedure/MasterProcedureScheduler.java | 113 ++-
.../procedure/PeerProcedureInterface.java | 34
.../hbase/master/procedure/PeerQueue.java | 54 +
.../master/procedure/RSProcedureDispatcher.java | 101 ++
.../hbase/master/procedure/SchemaLocking.java | 5 +
.../master/replication/ModifyPeerProcedure.java | 127
.../master/replication/RefreshPeerCallable.java | 67 +++
.../replication/RefreshPeerProcedure.java | 197 +++
.../hbase/procedure2/RSProcedureCallable.java | 43
.../hbase/regionserver/HRegionServer.java | 61 ++
.../hbase/regionserver/RSRpcServices.java | 58 --
.../handler/RSProcedureHandler.java | 51 +
.../assignment/TestAssignmentManager.java | 20 +-
.../replication/DummyModifyPeerProcedure.java | 41
.../TestDummyModifyPeerProcedure.java | 80
.../security/access/TestAccessController.java | 1 +
26 files changed, 1109 insertions(+), 115 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/25392608/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index c5fe62b..dc9b5d4 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,5 +22,5 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public enum LockedResourceType {
- SERVER, NAMESPACE, TABLE, REGION
+ SERVER, NAMESPACE, TABLE, REGION, PEER
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/25392608/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 6238e10..7e3dde6 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -226,13 +226,30 @@ public abstract class RemoteProcedureDispatcher
- * @param
*/
public interface RemoteProcedure {
+/**
+ * For building the remote operation.
+ */
RemoteOperation remoteCallBuild(TEnv env, TRemote remote);
-void remoteCallCompleted(TEnv env, TRemote remote, RemoteOperation
response);
+
+/**
+ * Called when the executeProcedure call is failed.
+ */
void remoteCallFailed(TEnv env, TRemote remote, IOException exception);
+
+/**
+ * Called when RS tells the remote procedure is succeeded through the
+ * {@code reportProcedureDone} method.
+ */
+void remoteOperationCompleted(TEnv env);
+
+/**
+ * Called when RS tells the remote procedure is failed through the {@code
reportProcedureDone}
+ * method.
+ * @param error the error message
+ */
+void remoteOperationFailed(TEnv env, String error);
[06/50] [abbrv] hbase git commit: HBASE-20110 Findbugs in zk and mr caused nightly #409 branch-2 to fail
HBASE-20110 Findbugs in zk and mr caused nightly #409 branch-2 to fail
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/03f9cb89
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/03f9cb89
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/03f9cb89
Branch: refs/heads/HBASE-19397-branch-2
Commit: 03f9cb89a62de38abf2402869807be5d42813b82
Parents: e0ff595
Author: Michael Stack
Authored: Wed Feb 28 23:38:35 2018 -0800
Committer: Michael Stack
Committed: Wed Feb 28 23:38:35 2018 -0800
--
.../src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java | 2 ++
.../org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java | 3 ---
2 files changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/03f9cb89/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index c8863bd..7c02be0 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -218,6 +218,8 @@ public class Import extends Configured implements Tool {
}
@Override
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"EQ_COMPARETO_USE_OBJECT_EQUALS",
+justification = "This is wrong, yes, but we should be purging
Writables, not fixing them")
public int compareTo(CellWritableComparable o) {
return CellComparator.getInstance().compare(this.kv, o.kv);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/03f9cb89/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
--
diff --git
a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 0285a0d..9343ec2 100644
---
a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++
b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -419,7 +419,6 @@ public class RecoverableZooKeeper {
// try to verify whether the previous setData success or not
try{
Stat stat = new Stat();
- startTime = EnvironmentEdgeManager.currentTime();
byte[] revData = checkZk().getData(path, false, stat);
if(Bytes.compareTo(revData, newData) == 0) {
// the bad version is caused by previous successful setData
@@ -556,7 +555,6 @@ public class RecoverableZooKeeper {
// If the connection was lost, there is still a possibility that
// we have successfully created the node at our previous attempt,
// so we read the node and compare.
- startTime = EnvironmentEdgeManager.currentTime();
byte[] currentData = checkZk().getData(path, false, null);
if (currentData != null &&
Bytes.compareTo(currentData, data) == 0) {
@@ -725,7 +723,6 @@ public class RecoverableZooKeeper {
}
public void sync(String path, AsyncCallback.VoidCallback cb, Object ctx)
throws KeeperException {
-long startTime = EnvironmentEdgeManager.currentTime();
checkZk().sync(path, cb, null);
}
[10/50] [abbrv] hbase git commit: HBASE-20113 Move branch-2 version from 2.0.0-beta-2-SNAPSHOT to 2.0.0-beta-2
HBASE-20113 Move branch-2 version from 2.0.0-beta-2-SNAPSHOT to 2.0.0-beta-2 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2de2956 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2de2956 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2de2956 Branch: refs/heads/HBASE-19397-branch-2 Commit: a2de29560f0f97e1e17c8e17f0a1ff548bdf473c Parents: 4547edb Author: Michael Stack Authored: Thu Mar 1 15:46:38 2018 -0800 Committer: Michael Stack Committed: Thu Mar 1 15:46:38 2018 -0800 -- hbase-annotations/pom.xml| 2 +- hbase-archetypes/hbase-archetype-builder/pom.xml | 2 +- hbase-archetypes/hbase-client-project/pom.xml| 2 +- hbase-archetypes/hbase-shaded-client-project/pom.xml | 2 +- hbase-archetypes/pom.xml | 2 +- hbase-assembly/pom.xml | 2 +- hbase-build-configuration/pom.xml| 2 +- hbase-build-support/hbase-error-prone/pom.xml| 4 ++-- hbase-build-support/pom.xml | 2 +- hbase-checkstyle/pom.xml | 4 ++-- hbase-client/pom.xml | 2 +- hbase-common/pom.xml | 2 +- hbase-endpoint/pom.xml | 2 +- hbase-examples/pom.xml | 2 +- hbase-external-blockcache/pom.xml| 2 +- hbase-hadoop-compat/pom.xml | 2 +- hbase-hadoop2-compat/pom.xml | 2 +- hbase-http/pom.xml | 2 +- hbase-it/pom.xml | 2 +- hbase-mapreduce/pom.xml | 2 +- hbase-metrics-api/pom.xml| 2 +- hbase-metrics/pom.xml| 2 +- hbase-procedure/pom.xml | 2 +- hbase-protocol-shaded/pom.xml| 2 +- hbase-protocol/pom.xml | 2 +- hbase-replication/pom.xml| 2 +- hbase-resource-bundle/pom.xml| 2 +- hbase-rest/pom.xml | 2 +- hbase-rsgroup/pom.xml| 2 +- hbase-server/pom.xml | 2 +- hbase-shaded/hbase-shaded-check-invariants/pom.xml | 2 +- hbase-shaded/hbase-shaded-client/pom.xml | 2 +- hbase-shaded/hbase-shaded-mapreduce/pom.xml | 2 +- hbase-shaded/pom.xml | 2 +- hbase-shell/pom.xml | 2 +- hbase-testing-util/pom.xml | 2 +- hbase-thrift/pom.xml | 2 +- hbase-zookeeper/pom.xml | 2 +- pom.xml | 2 +- 39 files changed, 41 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/a2de2956/hbase-annotations/pom.xml -- diff --git a/hbase-annotations/pom.xml b/hbase-annotations/pom.xml index bcce51c..7e16254 100644 --- a/hbase-annotations/pom.xml +++ b/hbase-annotations/pom.xml @@ -23,7 +23,7 @@ hbase org.apache.hbase -2.0.0-beta-2-SNAPSHOT +2.0.0-beta-2 .. http://git-wip-us.apache.org/repos/asf/hbase/blob/a2de2956/hbase-archetypes/hbase-archetype-builder/pom.xml -- diff --git a/hbase-archetypes/hbase-archetype-builder/pom.xml b/hbase-archetypes/hbase-archetype-builder/pom.xml index 49a0f8b..059d849 100644 --- a/hbase-archetypes/hbase-archetype-builder/pom.xml +++ b/hbase-archetypes/hbase-archetype-builder/pom.xml @@ -25,7 +25,7 @@ hbase-archetypes org.apache.hbase -2.0.0-beta-2-SNAPSHOT +2.0.0-beta-2 .. http://git-wip-us.apache.org/repos/asf/hbase/blob/a2de2956/hbase-archetypes/hbase-client-project/pom.xml -- diff --git a/hbase-archetypes/hbase-client-project/pom.xml b/hbase-archetypes/hbase-client-project/pom.xml index 35a2dbc..9cc803b 100644 --- a/hbase-archetypes/hbase-client-project/pom.xml +++ b/hbase-archetypes/hbase-client-project/pom.xml @@ -26,7 +26,7 @@ hbase-archetypes org.apache.hbase -2.0.0-beta-2-SNAPSHOT +2.0.0-beta-2 .. hbase-client-project http://git-wip-us.apache.org/repos/asf/hbase/blob/a2de2956/hbase-archetypes/hbase-shaded-client-project/pom.xml -- diff --git a/hbase-archetypes/hbase-shaded-client-project/pom.xml b/hbase-archetypes/hbase-shaded-clien
[46/50] [abbrv] hbase git commit: HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint
UT hangs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/040b8718
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/040b8718
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/040b8718
Branch: refs/heads/HBASE-19397-branch-2
Commit: 040b8718fed7ec8a2e97fca7e1e371b31ca09415
Parents: 5ee4e18
Author: huzheng
Authored: Wed Jan 10 15:00:30 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/040b8718/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 71af62b..cd967ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -40,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -71,6 +70,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -3407,13 +3407,12 @@ public class HMaster extends HRegionServer implements
MasterServices {
cpHost.preGetReplicationPeerConfig(peerId);
}
LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" +
peerId);
-Optional peerConfig =
- this.replicationPeerManager.getPeerConfig(peerId);
-
+ReplicationPeerConfig peerConfig =
this.replicationPeerManager.getPeerConfig(peerId)
+.orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
if (cpHost != null) {
cpHost.postGetReplicationPeerConfig(peerId);
}
-return peerConfig.orElse(null);
+return peerConfig;
}
@Override
[13/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
HBASE-19543 Abstract a replication storage interface to extract the zk specific
code
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fe4c0f64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fe4c0f64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fe4c0f64
Branch: refs/heads/HBASE-19397-branch-2
Commit: fe4c0f64a61278afff78c9c3e4f9f899861c5b82
Parents: 5d0e061
Author: zhangduo
Authored: Fri Dec 22 14:37:28 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../hadoop/hbase/util/CollectionUtils.java | 3 +
hbase-replication/pom.xml | 12 +
.../replication/ReplicationPeerStorage.java | 74
.../replication/ReplicationQueueStorage.java| 164 +++
.../replication/ReplicationStateZKBase.java | 1 -
.../replication/ReplicationStorageFactory.java | 49 +++
.../replication/ZKReplicationPeerStorage.java | 164 +++
.../replication/ZKReplicationQueueStorage.java | 425 +++
.../replication/ZKReplicationStorageBase.java | 75
.../TestZKReplicationPeerStorage.java | 171
.../TestZKReplicationQueueStorage.java | 171
.../org/apache/hadoop/hbase/master/HMaster.java | 34 +-
.../hadoop/hbase/master/MasterServices.java | 6 +-
.../master/procedure/MasterProcedureEnv.java| 24 +-
.../master/replication/AddPeerProcedure.java| 6 +-
.../replication/DisablePeerProcedure.java | 7 +-
.../master/replication/EnablePeerProcedure.java | 6 +-
.../master/replication/ModifyPeerProcedure.java | 41 +-
.../master/replication/RemovePeerProcedure.java | 6 +-
.../master/replication/ReplicationManager.java | 199 -
.../replication/ReplicationPeerManager.java | 331 +++
.../replication/UpdatePeerConfigProcedure.java | 7 +-
.../replication/TestReplicationAdmin.java | 64 ++-
.../hbase/master/MockNoopMasterServices.java| 13 +-
.../hbase/master/TestMasterNoCluster.java | 3 +-
.../TestReplicationDisableInactivePeer.java | 6 +-
26 files changed, 1749 insertions(+), 313 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe4c0f64/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
index 875b124..8bbb6f1 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
@@ -107,6 +107,9 @@ public class CollectionUtils {
return list.get(list.size() - 1);
}
+ public static List nullToEmpty(List list) {
+return list != null ? list : Collections.emptyList();
+ }
/**
* In HBASE-16648 we found that ConcurrentHashMap.get is much faster than
computeIfAbsent if the
* value already exists. Notice that the implementation does not guarantee
that the supplier will
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe4c0f64/hbase-replication/pom.xml
--
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index b459c3a..4a86c36 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -104,6 +104,18 @@
org.apache.hbase
hbase-zookeeper
+
+ org.apache.hbase
+ hbase-common
+ test-jar
+ test
+
+
+ org.apache.hbase
+ hbase-zookeeper
+ test-jar
+ test
+
org.apache.commons
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe4c0f64/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
new file mode 100644
index 000..e00cd0d
--- /dev/null
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *
[50/50] [abbrv] hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
HBASE-20082 Fix findbugs errors only on master which are introduced by
HBASE-19397
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5202f293
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5202f293
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5202f293
Branch: refs/heads/HBASE-19397-branch-2
Commit: 5202f293d43802938afaa25ee398ea4212ca95be
Parents: 3db22e6
Author: zhangduo
Authored: Mon Feb 26 10:22:09 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++
.../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5202f293/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 1253ef9..ba9bcdc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends
Procedure
private PeerOperationType type;
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"IS2_INCONSISTENT_SYNC",
+ justification = "Will never change after construction")
private ServerName targetServer;
private boolean dispatched;
http://git-wip-us.apache.org/repos/asf/hbase/blob/5202f293/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 6965f55..d11dc8e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -402,7 +402,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
toStartup.add(replicationSource);
}
}
-for (ReplicationSourceInterface replicationSource : oldsources) {
+for (ReplicationSourceInterface replicationSource : toStartup) {
replicationSource.startup();
}
}
[18/50] [abbrv] hbase git commit: HBASE-19579 Add peer lock test for shell command list_locks
HBASE-19579 Add peer lock test for shell command list_locks
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7674225c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7674225c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7674225c
Branch: refs/heads/HBASE-19397-branch-2
Commit: 7674225c7f79c29b71f492e1d1dc717d6d6a1149
Parents: 04c1ab8
Author: Guanghao Zhang
Authored: Sat Dec 23 21:04:27 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../src/main/protobuf/LockService.proto | 1 +
.../src/test/ruby/shell/list_locks_test.rb | 19 +++
2 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7674225c/hbase-protocol-shaded/src/main/protobuf/LockService.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
index b8d180c..0675070 100644
--- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
@@ -77,6 +77,7 @@ enum LockedResourceType {
NAMESPACE = 2;
TABLE = 3;
REGION = 4;
+ PEER = 5;
}
message LockedResource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7674225c/hbase-shell/src/test/ruby/shell/list_locks_test.rb
--
diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index f465a6b..ef1c0ce 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -67,6 +67,25 @@ module Hbase
proc_id)
end
+define_test 'list peer locks' do
+ lock = create_exclusive_lock(0)
+ peer_id = '1'
+
+ @scheduler.waitPeerExclusiveLock(lock, peer_id)
+ output = capture_stdout { @list_locks.command }
+ @scheduler.wakePeerExclusiveLock(lock, peer_id)
+
+ assert_equal(
+"PEER(1)\n" \
+"Lock type: EXCLUSIVE, procedure: {" \
+
"\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
+ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\",
\"state\"=>\"RUNNABLE\", " \
+ "\"lastUpdate\"=>\"0\", " \
+ "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\",
\"description\"=>\"description\"}]" \
+"}\n\n",
+output)
+end
+
define_test 'list server locks' do
lock = create_exclusive_lock(0)
[33/50] [abbrv] hbase git commit: HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bc04ee85
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bc04ee85
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bc04ee85
Branch: refs/heads/HBASE-19397-branch-2
Commit: bc04ee85a6a5c3535d449169b7a748c657aafe8d
Parents: e60f15f
Author: huzheng
Authored: Fri Dec 29 15:55:28 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 5 +-
.../replication/ReplicationStateZKBase.java | 159 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +--
.../replication/ZKReplicationPeerStorage.java | 24 ++-
.../replication/ZKReplicationStorageBase.java | 13 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
.../master/ReplicationPeerConfigUpgrader.java | 128 +++
.../regionserver/DumpReplicationQueues.java | 18 +--
.../replication/regionserver/Replication.java | 3 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +-
.../TestReplicationTrackerZKImpl.java | 3 +-
.../replication/master/TestTableCFsUpdater.java | 41 ++---
.../TestReplicationSourceManager.java | 6 +-
13 files changed, 136 insertions(+), 292 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc04ee85/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c66aff..2a970ba 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -33,9 +33,8 @@ public class ReplicationFactory {
return new ReplicationPeers(zk, conf);
}
- public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable
abortable,
+ public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
Abortable abortable,
Stoppable stopper) {
-return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf,
abortable, stopper);
+return new ReplicationTrackerZKImpl(zookeeper, abortable, stopper);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc04ee85/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
deleted file mode 100644
index a48683e..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audie
[40/50] [abbrv] hbase git commit: HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dbc81ff5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dbc81ff5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dbc81ff5
Branch: refs/heads/HBASE-19397-branch-2
Commit: dbc81ff57a0969dc172b519c81fcbc67d91c9c75
Parents: f5b151f
Author: zhangduo
Authored: Wed Dec 27 22:03:51 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 9 +-
.../hbase/replication/ReplicationQueues.java| 161 ---
.../replication/ReplicationQueuesArguments.java | 70 ---
.../replication/ReplicationQueuesZKImpl.java| 417 -
.../hbase/replication/ReplicationTableBase.java | 442 ---
.../replication/ReplicationTrackerZKImpl.java | 21 +-
.../replication/ZKReplicationQueueStorage.java | 22 +
.../replication/TestReplicationStateBasic.java | 131 +++---
.../replication/TestReplicationStateZKImpl.java | 41 +-
.../regionserver/DumpReplicationQueues.java | 15 +-
.../RecoveredReplicationSource.java | 17 +-
.../RecoveredReplicationSourceShipper.java | 22 +-
.../replication/regionserver/Replication.java | 20 +-
.../regionserver/ReplicationSource.java | 16 +-
.../ReplicationSourceInterface.java | 11 +-
.../regionserver/ReplicationSourceManager.java | 265 ++-
.../regionserver/ReplicationSyncUp.java | 13 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 12 +-
.../cleaner/TestReplicationHFileCleaner.java| 23 +-
.../cleaner/TestReplicationZKNodeCleaner.java | 22 +-
.../replication/ReplicationSourceDummy.java | 6 +-
.../replication/TestReplicationSyncUpTool.java | 4 +-
.../TestReplicationSourceManager.java | 97 ++--
.../TestReplicationSourceManagerZkImpl.java | 57 +--
24 files changed, 356 insertions(+), 1558 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbc81ff5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c1c213..5e70e57 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.commons.lang3.reflect.ConstructorUtils;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A factory class for instantiating replication objects that deal with
replication state.
@@ -30,12 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationQueues
getReplicationQueues(ReplicationQueuesArguments args)
- throws Exception {
-return (ReplicationQueues)
ConstructorUtils.invokeConstructor(ReplicationQueuesZKImpl.class,
- args);
- }
-
public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbc81ff5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
deleted file mode 100644
index a2d21f7..000
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apac
[05/50] [abbrv] hbase git commit: HBASE-20107 Add a test case for HBASE-14317 (Zephyr Guo)
HBASE-20107 Add a test case for HBASE-14317 (Zephyr Guo)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0ff5957
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0ff5957
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0ff5957
Branch: refs/heads/HBASE-19397-branch-2
Commit: e0ff59574466accc226bd2e8c2d32c37bdde129c
Parents: 313464f
Author: tedyu
Authored: Wed Feb 28 21:18:14 2018 -0800
Committer: tedyu
Committed: Wed Feb 28 21:18:14 2018 -0800
--
.../hadoop/hbase/regionserver/wal/FSHLog.java | 6 +-
.../hbase/regionserver/TestWALLockup.java | 135 +++
2 files changed, 139 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0ff5957/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index a2aabc8..b1ae1dc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -679,7 +679,8 @@ public class FSHLog extends AbstractFSWAL {
return logRollNeeded;
}
- private long getSequenceOnRingBuffer() {
+ @VisibleForTesting
+ protected long getSequenceOnRingBuffer() {
return this.disruptor.getRingBuffer().next();
}
@@ -688,7 +689,8 @@ public class FSHLog extends AbstractFSWAL {
return publishSyncOnRingBuffer(sequence);
}
- private SyncFuture publishSyncOnRingBuffer(long sequence) {
+ @VisibleForTesting
+ protected SyncFuture publishSyncOnRingBuffer(long sequence) {
// here we use ring buffer sequence as transaction id
SyncFuture syncFuture = getSyncFuture(sequence);
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0ff5957/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 788a708..d6a6079 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.wal.WALProvider.Writer;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
@@ -308,6 +309,140 @@ public class TestWALLockup {
}
}
+ /**
+ *
+ * If below is broken, we will see this test timeout because
RingBufferEventHandler was stuck in
+ * attainSafePoint. Everyone will wait for sync to finish forever. See
HBASE-14317.
+ */
+ @Test (timeout=3)
+ public void testRingBufferEventHandlerStuckWhenSyncFailed()
+throws IOException, InterruptedException {
+
+// A WAL that we can have throw exceptions and slow FSHLog.replaceWriter
down
+class DodgyFSLog extends FSHLog {
+
+ private volatile boolean zigZagCreated = false;
+
+ public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration
conf)
+throws IOException {
+super(fs, root, logDir, conf);
+ }
+
+ @Override
+ protected void afterCreatingZigZagLatch() {
+zigZagCreated = true;
+// Sleep a while to wait for RingBufferEventHandler to get stuck first.
+try {
+ Thread.sleep(3000);
+} catch (InterruptedException ignore) {
+}
+ }
+
+ @Override
+ protected long getSequenceOnRingBuffer() {
+return super.getSequenceOnRingBuffer();
+ }
+
+ protected void publishSyncOnRingBufferAndBlock(long sequence) {
+try {
+ super.blockOnSync(super.publishSyncOnRingBuffer(sequence));
+ Assert.fail("Expect an IOException here.");
+} catch (IOException ignore) {
+ // Here, we will get an IOException.
+}
+ }
+
+ @Override
+ protected Writer createWriterInstance(Path path) throws IOException {
+final Writer w = super.createWriterInstance(path);
+return new Writer() {
+ @Override
+ public void close() throws IOException {
+w.close();
+ }
+
+ @Override
+ public void sync() throws IOException {
+throw new IOException("FAKE! Failed to replace a bad
datanode.
[20/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/04c1ab86
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/04c1ab86
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/04c1ab86
Branch: refs/heads/HBASE-19397-branch-2
Commit: 04c1ab869192eb3f209926a08beac22f38ca9d3a
Parents: fe4c0f6
Author: zhangduo
Authored: Mon Dec 25 18:49:56 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../hbase/replication/ReplicationFactory.java | 19 +-
.../replication/ReplicationPeersZKImpl.java | 21 +-
.../replication/ReplicationQueueStorage.java| 26 +-
.../replication/ReplicationQueuesClient.java| 93 -
.../ReplicationQueuesClientArguments.java | 40 --
.../ReplicationQueuesClientZKImpl.java | 181 -
.../replication/ZKReplicationQueueStorage.java | 90 -
.../replication/TestReplicationStateBasic.java | 378 +++
.../replication/TestReplicationStateZKImpl.java | 153
.../TestZKReplicationQueueStorage.java | 74
.../cleaner/ReplicationZKNodeCleaner.java | 71 ++--
.../cleaner/ReplicationZKNodeCleanerChore.java | 5 +-
.../replication/ReplicationPeerManager.java | 31 +-
.../master/ReplicationHFileCleaner.java | 109 ++
.../master/ReplicationLogCleaner.java | 44 +--
.../regionserver/DumpReplicationQueues.java | 78 ++--
.../hbase/util/hbck/ReplicationChecker.java | 14 +-
.../client/TestAsyncReplicationAdminApi.java| 31 +-
.../replication/TestReplicationAdmin.java | 2 +
.../hbase/master/cleaner/TestLogsCleaner.java | 81 ++--
.../cleaner/TestReplicationHFileCleaner.java| 29 --
.../cleaner/TestReplicationZKNodeCleaner.java | 12 +-
.../replication/TestReplicationStateBasic.java | 378 ---
.../replication/TestReplicationStateZKImpl.java | 232
.../TestReplicationSourceManagerZkImpl.java | 41 --
25 files changed, 890 insertions(+), 1343 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/04c1ab86/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 9f4ad18..6c1c213 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,20 +36,14 @@ public class ReplicationFactory {
args);
}
- public static ReplicationQueuesClient
- getReplicationQueuesClient(ReplicationQueuesClientArguments args) throws
Exception {
-return (ReplicationQueuesClient) ConstructorUtils
-.invokeConstructor(ReplicationQueuesClientZKImpl.class, args);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- Abortable abortable) {
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
}
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk,
Configuration conf,
- final
ReplicationQueuesClient queuesClient, Abortable abortable) {
-return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk,
Configuration conf,
+ ReplicationQueueStorage queueStorage, Abortable abortable) {
+return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable);
}
public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
http://git-wip-us.apache.org/repos/asf/hbase/blob/04c1ab86/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 419e289..4e5f757 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hb
[03/50] [abbrv] hbase git commit: HBASE-20106 [api compliance chacker] Fix Bug Where Branch Isn't Found
HBASE-20106 [api compliance chacker] Fix Bug Where Branch Isn't Found While git rev-parse, sometimes the branch cannot be found unless the remote is specified. This fix tries to use "origin" if the remote is not specified and the branch is not found. Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2f3632a2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2f3632a2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2f3632a2 Branch: refs/heads/HBASE-19397-branch-2 Commit: 2f3632a2a47cc7a24f475460532fc14b0cf47254 Parents: f71c00c Author: Alex Leblang Authored: Thu Sep 7 12:31:53 2017 -0400 Committer: Sean Busbey Committed: Wed Feb 28 11:54:31 2018 -0600 -- dev-support/checkcompatibility.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/2f3632a2/dev-support/checkcompatibility.py -- diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py index 83189cf..ea9c229 100755 --- a/dev-support/checkcompatibility.py +++ b/dev-support/checkcompatibility.py @@ -116,7 +116,12 @@ def checkout_java_tree(rev, path): def get_git_hash(revname): """ Convert 'revname' to its SHA-1 hash. """ -return check_output(["git", "rev-parse", revname], +try: +return check_output(["git", "rev-parse", revname], +cwd=get_repo_dir()).strip() +except: +revname = "origin/" + revname +return check_output(["git", "rev-parse", revname], cwd=get_repo_dir()).strip()
[35/50] [abbrv] hbase git commit: HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
http://git-wip-us.apache.org/repos/asf/hbase/blob/d5277a27/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
index 4c1b94f..6d2b578 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
@@ -28,12 +28,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -41,6 +40,8 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
@@ -71,9 +72,6 @@ public class TestNamespaceReplication extends
TestReplicationBase {
private static final byte[] val = Bytes.toBytes("myval");
- private static HTableDescriptor tabA;
- private static HTableDescriptor tabB;
-
private static Connection connection1;
private static Connection connection2;
private static Admin admin1;
@@ -93,23 +91,21 @@ public class TestNamespaceReplication extends
TestReplicationBase {
admin2.createNamespace(NamespaceDescriptor.create(ns1).build());
admin2.createNamespace(NamespaceDescriptor.create(ns2).build());
-tabA = new HTableDescriptor(tabAName);
-HColumnDescriptor fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabA.addFamily(fam);
+TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(tabAName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabA = builder.build();
admin1.createTable(tabA);
admin2.createTable(tabA);
-tabB = new HTableDescriptor(tabBName);
-fam = new HColumnDescriptor(f1Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
-fam = new HColumnDescriptor(f2Name);
-fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
-tabB.addFamily(fam);
+builder = TableDescriptorBuilder.newBuilder(tabBName);
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+TableDescriptor tabB = builder.build();
admin1.createTable(tabB);
admin2.createTable(tabB);
}
@@ -137,22 +133,24 @@ public class TestNamespaceReplication extends
TestReplicationBase {
@Test
public void testNamespaceReplication() throws Exception {
+String peerId = "2";
+
Table htab1A = connection1.getTable(tabAName);
Table htab2A = connection2.getTable(tabAName);
Table htab1B = connection1.getTable(tabBName);
Table htab2B = connection2.getTable(tabBName);
-ReplicationPeerConfig rpc = admin.getPeerConfig("2");
-rpc.setReplicateAllUserTables(false);
-admin.updatePeerConfig("2", rpc);
+ReplicationPeerConfig rpc = admin1.getReplicationPeerConfig(peerId);
+admin1.updateReplicationPeerConfig(peerId,
+
ReplicationPeerConfig.newBuilder(rpc).setReplicateAllUserTables(false).build());
// add ns1 to peer config which replicate to cluster2
-rpc = admin.getPeerConfig("2");
+rpc = admin1.getReplicationPeerConfig(peerId);
Set namespaces = new HashSet<>();
namespaces.add(ns1);
-rpc.setNamespaces(namespaces);
-admin.updatePeerConfig("2", rpc);
+admin1.updateReplicationPeerConfig(peerId,
+ Replication
[19/50] [abbrv] hbase git commit: HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
http://git-wip-us.apache.org/repos/asf/hbase/blob/04c1ab86/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 6e27a21..d8f9625 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,13 +21,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,17 +48,18 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
@@ -303,57 +304,53 @@ public class DumpReplicationQueues extends Configured
implements Tool {
}
public String dumpQueues(ClusterConnection connection, ZKWatcher zkw,
Set peerIds,
- boolean hdfs) throws Exception {
-ReplicationQueuesClient queuesClient;
+ boolean hdfs) throws Exception {
+ReplicationQueueStorage queueStorage;
ReplicationPeers replicationPeers;
ReplicationQueues replicationQueues;
ReplicationTracker replicationTracker;
-ReplicationQueuesClientArguments replicationArgs =
-new ReplicationQueuesClientArguments(getConf(), new
WarnOnlyAbortable(), zkw);
+ReplicationQueuesArguments replicationArgs =
+new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(),
zkw);
StringBuilder sb = new StringBuilder();
-queuesClient =
ReplicationFactory.getReplicationQueuesClient(replicationArgs);
-queuesClient.init();
+queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw,
getConf());
replicationQueues =
ReplicationFactory.getReplicationQueues(replicationArgs);
-replicationPeers = ReplicationFactory.getReplicationPeers(zkw, getConf(),
queuesClient, connection);
+replicationPeers =
+ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage,
connection);
replicationTracker = ReplicationFactory.getReplicationTracker(zkw,
replicationPeers, getConf(),
new WarnOnlyAbortable(), new WarnOnlyStoppable());
-List liveRegionServers =
replicationTracker.getListOfRegionServers();
+Set liveRegionServers = new
HashSet<>(replicationTracker.getListOfRegionServers());
// Loops each peer on each RS and dumps the queues
-try {
- List regionservers = queuesClient.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
-return sb.toString();
+List regionservers = queueStorage.getListOfReplicators();
+if (regionservers == null || regionservers.isEmpty()) {
+ return sb.toString();
+}
+for (ServerName regionserver : regionservers) {
+ List queueIds = queueStorage.getAllQueues(regionserver);
+ replicationQueues.init(regionserver.getServerName());
+ if (!liveRegionServers.contains(regionserver.getServerName())) {
+deadRegionServers.add(regionserver.getServerName());
}
- for (String regionserver : regionservers) {
-List queueIds = queuesClient.getAllQueues(regionserver);
-replicationQueues.init(regionserver);
-if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver);
-}
-for (String
[37/50] [abbrv] hbase git commit: HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/855ea3b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/855ea3b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/855ea3b6
Branch: refs/heads/HBASE-19397-branch-2
Commit: 855ea3b6962b7078fe92ffaca06fcabfc99069fd
Parents: 388fb4b
Author: Guanghao Zhang
Authored: Fri Jan 5 15:39:06 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 23
1 file changed, 19 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/855ea3b6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index b18dd6c..eab06a2 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -290,7 +290,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
@Override
public void completionCleanup(final Procedure proc) {
if (proc instanceof TableProcedureInterface) {
- TableProcedureInterface iProcTable = (TableProcedureInterface)proc;
+ TableProcedureInterface iProcTable = (TableProcedureInterface) proc;
boolean tableDeleted;
if (proc.hasException()) {
Exception procEx = proc.getException().unwrapRemoteException();
@@ -311,9 +311,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
}
} else if (proc instanceof PeerProcedureInterface) {
PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc;
- if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) {
-removePeerQueue(iProcPeer.getPeerId());
- }
+ tryCleanupPeerQueue(iProcPeer.getPeerId(), proc);
} else {
// No cleanup for ServerProcedureInterface types, yet.
return;
@@ -402,6 +400,23 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
locking.removePeerLock(peerId);
}
+ private void tryCleanupPeerQueue(String peerId, Procedure procedure) {
+schedLock();
+try {
+ PeerQueue queue = AvlTree.get(peerMap, peerId,
PEER_QUEUE_KEY_COMPARATOR);
+ if (queue == null) {
+return;
+ }
+
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
+removeFromRunQueue(peerRunQueue, queue);
+removePeerQueue(peerId);
+ }
+} finally {
+ schedUnlock();
+}
+ }
private static boolean isPeerProcedure(Procedure proc) {
return proc instanceof PeerProcedureInterface;
[09/50] [abbrv] hbase git commit: HBASE-20055 Removed declaration of un-thrown exceptions and unused setRegionStateBackToOpen() from MergeTableRegionsProcedure
HBASE-20055 Removed declaration of un-thrown exceptions and unused
setRegionStateBackToOpen() from MergeTableRegionsProcedure
Plus some minor cleanup.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4547edbf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4547edbf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4547edbf
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4547edbfe077ae0ce0ef665c27814404a110baa3
Parents: a08ade9
Author: Umesh Agashe
Authored: Thu Feb 22 16:29:52 2018 -0800
Committer: Michael Stack
Committed: Thu Mar 1 08:45:54 2018 -0800
--
.../assignment/MergeTableRegionsProcedure.java | 46 +---
.../TestRegionMergeTransactionOnCluster.java| 2 +-
2 files changed, 22 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4547edbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 7c041e7..052ba7f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -52,8 +52,6 @@ import
org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
-import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
-import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
@@ -146,7 +144,7 @@ public class MergeTableRegionsProcedure
}
if (!RegionInfo.areAdjacent(regionToMergeA, regionToMergeB)) {
- String msg = "Unable to merge not adjacent regions " +
regionToMergeA.getShortNameToLog() +
+ String msg = "Unable to merge non-adjacent regions " +
regionToMergeA.getShortNameToLog() +
", " + regionToMergeB.getShortNameToLog() + " where forcible = " +
forcible;
LOG.warn(msg);
if (!forcible) {
@@ -194,7 +192,7 @@ public class MergeTableRegionsProcedure
private static long getMergedRegionIdTimestamp(final RegionInfo
regionToMergeA,
final RegionInfo regionToMergeB) {
long rid = EnvironmentEdgeManager.currentTime();
-// Regionid is timestamp. Merged region's id can't be less than that of
+// Region Id is a timestamp. Merged region's id can't be less than that of
// merging regions else will insert at wrong location in hbase:meta (See
HBASE-710).
if (rid < regionToMergeA.getRegionId() || rid <
regionToMergeB.getRegionId()) {
LOG.warn("Clock skew; merging regions id are " +
regionToMergeA.getRegionId()
@@ -205,8 +203,8 @@ public class MergeTableRegionsProcedure
}
@Override
- protected Flow executeFromState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
- throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
+ protected Flow executeFromState(final MasterProcedureEnv env,
+ final MergeTableRegionsState state) {
LOG.trace("{} execute state={}", this, state);
try {
switch (state) {
@@ -260,7 +258,7 @@ public class MergeTableRegionsProcedure
RegionInfo.getShortNameToLog(regionsToMerge) + " in the table " +
getTableName() +
" (in state=" + state + ")";
if (!isRollbackSupported(state)) {
-// We reach a state that cannot be rolled back. We just need to keep
retry.
+// We reach a state that cannot be rolled back. We just need to keep
retrying.
LOG.warn(msg, e);
} else {
LOG.error(msg, e);
@@ -277,9 +275,8 @@ public class MergeTableRegionsProcedure
* See HBASE-19851 for details.
*/
@Override
- protected void rollbackState(
- final MasterProcedureEnv env,
- final MergeTableRegionsState state) throws IOException,
InterruptedException {
+ protected void rollbackState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
+ throws IOException {
if (isTraceE
[01/50] [abbrv] hbase git commit: HBASE-20084 Refactor the RSRpcServices#doBatchOp [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19397-branch-2 732d22793 -> 5202f293d (forced update)
HBASE-20084 Refactor the RSRpcServices#doBatchOp
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e50e6f7c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e50e6f7c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e50e6f7c
Branch: refs/heads/HBASE-19397-branch-2
Commit: e50e6f7ce92733472a757acee2c22d7fed329aef
Parents: c459282
Author: Chia-Ping Tsai
Authored: Mon Feb 26 20:49:05 2018 +0800
Committer: Chia-Ping Tsai
Committed: Wed Feb 28 15:13:09 2018 +0800
--
.../hbase/regionserver/RSRpcServices.java | 115 ++-
1 file changed, 58 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e50e6f7c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 42284e9..d0a1315 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -122,6 +122,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessChecker;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CollectionUtils;
import org.apache.hadoop.hbase.util.DNS;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
@@ -762,7 +763,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
// Gather up CONTIGUOUS Puts and Deletes in this mutations List. Idea is
that rather than do
// one at a time, we instead pass them in batch. Be aware that the
corresponding
// ResultOrException instance that matches each Put or Delete is then
added down in the
-// doBatchOp call. We should be staying aligned though the Put and Delete
are deferred/batched
+// doNonAtomicBatchOp call. We should be staying aligned though the Put
and Delete are
+// deferred/batched
List mutations = null;
long maxQuotaResultSize = Math.min(maxScannerResultSize,
quota.getReadAvailable());
IOException sizeIOE = null;
@@ -801,7 +803,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
// use it for the response.
//
// This will create a copy in the builder.
- hasResultOrException = true;
NameBytesPair pair = ResponseConverter.buildException(sizeIOE);
resultOrExceptionBuilder.setException(pair);
context.incrementResponseExceptionSize(pair.getSerializedSize());
@@ -828,29 +829,23 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
}
} else if (action.hasServiceCall()) {
hasResultOrException = true;
- try {
-com.google.protobuf.Message result =
-execServiceOnRegion(region, action.getServiceCall());
-ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder
=
-ClientProtos.CoprocessorServiceResult.newBuilder();
-resultOrExceptionBuilder.setServiceResult(
-serviceResultBuilder.setValue(
- serviceResultBuilder.getValueBuilder()
-.setName(result.getClass().getName())
-// TODO: Copy!!!
-
.setValue(UnsafeByteOperations.unsafeWrap(result.toByteArray();
- } catch (IOException ioe) {
-rpcServer.getMetrics().exception(ioe);
-NameBytesPair pair = ResponseConverter.buildException(ioe);
-resultOrExceptionBuilder.setException(pair);
-context.incrementResponseExceptionSize(pair.getSerializedSize());
- }
+ com.google.protobuf.Message result =
+execServiceOnRegion(region, action.getServiceCall());
+ ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder =
+ClientProtos.CoprocessorServiceResult.newBuilder();
+ resultOrExceptionBuilder.setServiceResult(
+serviceResultBuilder.setValue(
+ serviceResultBuilder.getValueBuilder()
+.setName(result.getClass().getName())
+// TODO: Copy!!!
+
.setValue(UnsafeByteOperations.unsafeWrap(result.toByteArray();
} else if (action.hasMutation()) {
MutationType type = action.getMutation().getMutateType();
[29/50] [abbrv] hbase git commit: HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore
HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker
and remove ReplicationZKNodeCleanerChore
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e60f15f5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e60f15f5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e60f15f5
Branch: refs/heads/HBASE-19397-branch-2
Commit: e60f15f557f4f5a72d178db3199171d2b17ef9c6
Parents: e39dbec
Author: zhangduo
Authored: Wed Jan 3 09:39:44 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../replication/VerifyReplication.java | 6 +-
.../hbase/replication/ReplicationPeers.java | 26 +--
.../hbase/replication/ReplicationUtils.java | 38
.../replication/TestReplicationStateBasic.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 13 --
.../cleaner/ReplicationZKNodeCleaner.java | 192 ---
.../cleaner/ReplicationZKNodeCleanerChore.java | 54 --
.../replication/ReplicationPeerManager.java | 18 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 11 +-
.../hbase/util/hbck/ReplicationChecker.java | 109 +++
.../cleaner/TestReplicationZKNodeCleaner.java | 115 ---
.../hbase/util/TestHBaseFsckReplication.java| 101 ++
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 6 +-
13 files changed, 226 insertions(+), 465 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e60f15f5/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index fe45762..fac4875 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -345,10 +345,10 @@ public class VerifyReplication extends Configured
implements Tool {
}
});
ReplicationPeerStorage storage =
- ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
+ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId);
return Pair.newPair(peerConfig,
-ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf));
+ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf));
} catch (ReplicationException e) {
throw new IOException("An error occurred while trying to connect to the
remove peer cluster",
e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e60f15f5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 45940a5..fcbc350 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -17,14 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -106,25 +103,6 @@ public class ReplicationPeers {
return Collections.unmodifiableSet(peerCache.keySet());
}
- public s
[32/50] [abbrv] hbase git commit: HBASE-19697 Remove TestReplicationAdminUsingProcedure
HBASE-19697 Remove TestReplicationAdminUsingProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/84514c0e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/84514c0e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/84514c0e
Branch: refs/heads/HBASE-19397-branch-2
Commit: 84514c0e19d83c04954ca40cf14a024e4400d2d5
Parents: bc04ee8
Author: zhangduo
Authored: Wed Jan 3 21:13:57 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../TestReplicationAdminUsingProcedure.java | 225 ---
1 file changed, 225 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/84514c0e/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
deleted file mode 100644
index 1300376..000
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client.replication;
-
-import java.io.IOException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ MediumTests.class, ClientTests.class })
-public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
-
- private static final String PEER_ID = "2";
- private static final Logger LOG =
Logger.getLogger(TestReplicationAdminUsingProcedure.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-conf1.setInt("hbase.multihconnection.threads.max", 10);
-
-// Start the master & slave mini cluster.
-TestReplicationBase.setUpBeforeClass();
-
-// Remove the replication peer
-hbaseAdmin.removeReplicationPeer(PEER_ID);
- }
-
- private void loadData(int startRowKey, int endRowKey) throws IOException {
-for (int i = startRowKey; i < endRowKey; i++) {
- byte[] rowKey = Bytes.add(row, Bytes.toBytes(i));
- Put put = new Put(rowKey);
- put.addColumn(famName, null, Bytes.toBytes(i));
- htable1.put(put);
-}
- }
-
- private void waitForReplication(int expectedRows, int retries)
- throws IOException, InterruptedException {
-Scan scan;
-for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i == retries - 1) {
-throw new IOException("Waited too much time for normal batch
replication");
- }
- try (ResultScanner scanner = htable2.getScanner(scan)) {
-int count = 0;
-for (Result res : scanner) {
- count++;
-}
-if (count != expectedRows) {
- LOG.info("Only got " + count + " rows, expected rows: " +
expectedRows);
- Thread.sleep(SLEEP_TIME);
-} else {
- return;
-}
- }
-}
- }
-
- @Before
- public void setUp() throws IOException {
-ReplicationPeerConfig rpc = new ReplicationPeerConfig();
-
[23/50] [abbrv] hbase git commit: HBASE-19642 Fix locking for peer modification procedure
HBASE-19642 Fix locking for peer modification procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f5b151fc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f5b151fc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f5b151fc
Branch: refs/heads/HBASE-19397-branch-2
Commit: f5b151fc42b5f61ecd7b406d1b2b12fa9f2f46a1
Parents: 2b80328
Author: zhangduo
Authored: Wed Dec 27 18:27:13 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../master/replication/ModifyPeerProcedure.java | 21 +---
1 file changed, 18 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f5b151fc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 279fbc7..a682606 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -46,6 +46,8 @@ public abstract class ModifyPeerProcedure
protected String peerId;
+ private volatile boolean locked;
+
// used to keep compatible with old client where we can only returns after
updateStorage.
protected ProcedurePrepareLatch latch;
@@ -145,17 +147,30 @@ public abstract class ModifyPeerProcedure
@Override
protected LockState acquireLock(MasterProcedureEnv env) {
-return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)
- ? LockState.LOCK_EVENT_WAIT
- : LockState.LOCK_ACQUIRED;
+if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+}
+locked = true;
+return LockState.LOCK_ACQUIRED;
}
@Override
protected void releaseLock(MasterProcedureEnv env) {
+locked = false;
env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
}
@Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+return true;
+ }
+
+ @Override
+ protected boolean hasLock(MasterProcedureEnv env) {
+return locked;
+ }
+
+ @Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState
state)
throws IOException, InterruptedException {
if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
[15/50] [abbrv] hbase git commit: HBASE-19520 Add UTs for the new lock type PEER
HBASE-19520 Add UTs for the new lock type PEER
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/69432ea7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/69432ea7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/69432ea7
Branch: refs/heads/HBASE-19397-branch-2
Commit: 69432ea72284d8c141de92502ad269a3fd6c4da9
Parents: 601818c
Author: Guanghao Zhang
Authored: Wed Dec 20 16:43:38 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../procedure/MasterProcedureScheduler.java | 9 +-
.../procedure/TestMasterProcedureScheduler.java | 65 -
...TestMasterProcedureSchedulerConcurrency.java | 135 +++
3 files changed, 201 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/69432ea7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 58f9ccd..b18dd6c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -277,6 +277,13 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
while (tableIter.hasNext()) {
count += tableIter.next().size();
}
+
+// Peer queues
+final AvlTreeIterator peerIter = new AvlTreeIterator<>(peerMap);
+while (peerIter.hasNext()) {
+ count += peerIter.next().size();
+}
+
return count;
}
@@ -807,7 +814,7 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
* @see #wakePeerExclusiveLock(Procedure, String)
* @param procedure the procedure trying to acquire the lock
* @param peerId peer to lock
- * @return true if the procedure has to wait for the per to be available
+ * @return true if the procedure has to wait for the peer to be available
*/
public boolean waitPeerExclusiveLock(Procedure procedure, String peerId) {
schedLock();
http://git-wip-us.apache.org/repos/asf/hbase/blob/69432ea7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index 160f4d2..65757db 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -908,6 +908,27 @@ public class TestMasterProcedureScheduler {
}
}
+ public static class TestPeerProcedure extends TestProcedure implements
PeerProcedureInterface {
+private final String peerId;
+private final PeerOperationType opType;
+
+public TestPeerProcedure(long procId, String peerId, PeerOperationType
opType) {
+ super(procId);
+ this.peerId = peerId;
+ this.opType = opType;
+}
+
+@Override
+public String getPeerId() {
+ return peerId;
+}
+
+@Override
+public PeerOperationType getPeerOperationType() {
+ return opType;
+}
+ }
+
private static LockProcedure createLockProcedure(LockType lockType, long
procId) throws Exception {
LockProcedure procedure = new LockProcedure();
@@ -930,22 +951,19 @@ public class TestMasterProcedureScheduler {
return createLockProcedure(LockType.SHARED, procId);
}
- private static void assertLockResource(LockedResource resource,
- LockedResourceType resourceType, String resourceName)
- {
+ private static void assertLockResource(LockedResource resource,
LockedResourceType resourceType,
+ String resourceName) {
assertEquals(resourceType, resource.getResourceType());
assertEquals(resourceName, resource.getResourceName());
}
- private static void assertExclusiveLock(LockedResource resource,
Procedure procedure)
- {
+ private static void assertExclusiveLock(LockedResource resource,
Procedure procedure) {
assertEquals(LockType.EXCLUSIVE, resource.getLockType());
assertEquals(procedure, resource.getExclusiveLockOwnerProcedure());
assertEquals(0, resource.getSharedLockCount());
}
- private static void assertSharedLock(LockedResource resource, int lockCount)
-
[48/50] [abbrv] hbase git commit: HBASE-19873 addendum add missing rule for new tests
HBASE-19873 addendum add missing rule for new tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f6727de0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f6727de0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f6727de0
Branch: refs/heads/HBASE-19397-branch-2
Commit: f6727de02f8907bffb849f140bba654f8d004706
Parents: 557c444
Author: zhangduo
Authored: Tue Jan 30 09:40:23 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 15:00:51 2018 +0800
--
.../hbase/replication/TestZKReplicationPeerStorage.java | 7 ++-
.../hbase/replication/TestZKReplicationQueueStorage.java | 8 +++-
.../hbase/replication/TestReplicationProcedureRetry.java | 7 ++-
.../apache/hadoop/hbase/util/TestHBaseFsckReplication.java | 6 ++
4 files changed, 25 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6727de0/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
index 3eb11da..3290fb0 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -33,19 +33,24 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Stream;
-
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationPeerStorage {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationPeerStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationPeerStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6727de0/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
--
diff --git
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
index 786730f..2c01a26 100644
---
a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++
b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -27,8 +27,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
-
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -38,11 +38,17 @@ import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestZKReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
+
private static final HBaseZKTestingUtility UTIL = new
HBaseZKTestingUtility();
private static ZKReplicationQueueStorage STORAGE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6727de0/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
index ab35b46..a2ae0b4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
[34/50] [abbrv] hbase git commit: HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45ee0d5f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45ee0d5f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45ee0d5f
Branch: refs/heads/HBASE-19397-branch-2
Commit: 45ee0d5f99721e102e37e87cf2e1505079e0c4b6
Parents: ae8a029
Author: zhangduo
Authored: Tue Jan 2 16:13:55 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:57:02 2018 +0800
--
.../regionserver/PeerProcedureHandlerImpl.java | 41 ++--
1 file changed, 29 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/45ee0d5f/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 1efe180..c09c6a0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import java.util.concurrent.locks.ReentrantLock;
-
+import java.util.concurrent.locks.Lock;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.KeyLocker;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +32,7 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
private static final Logger LOG =
LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private final ReplicationSourceManager replicationSourceManager;
- private final ReentrantLock peersLock = new ReentrantLock();
+ private final KeyLocker peersLock = new KeyLocker<>();
public PeerProcedureHandlerImpl(ReplicationSourceManager
replicationSourceManager) {
this.replicationSourceManager = replicationSourceManager;
@@ -40,40 +40,57 @@ public class PeerProcedureHandlerImpl implements
PeerProcedureHandler {
@Override
public void addPeer(String peerId) throws ReplicationException, IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
replicationSourceManager.addPeer(peerId);
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void removePeer(String peerId) throws ReplicationException,
IOException {
-peersLock.lock();
+Lock peerLock = peersLock.acquireLock(peerId);
try {
if (replicationSourceManager.getReplicationPeers().getPeer(peerId) !=
null) {
replicationSourceManager.removePeer(peerId);
}
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void disablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("disable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("disable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException,
IOException {
-PeerState newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
-LOG.info("enable replication peer, id: " + peerId + ", new state: " +
newState);
+PeerState newState;
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ newState =
replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+} finally {
+ peerLock.unlock();
+}
+LOG.info("enable replication peer, id: {}, new state: {}", peerId,
newState);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException,
IOException {
-replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+Lock peerLock = peersLock.acquireLock(peerId);
+try {
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+} finally {
+ peerLock.unlock()
[12/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication storage interface to extract the zk specific code
http://git-wip-us.apache.org/repos/asf/hbase/blob/fe4c0f64/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
deleted file mode 100644
index b6f8784..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Manages and performs all replication admin operations.
- *
- * Used to add/remove a replication peer.
- */
-@InterfaceAudience.Private
-public class ReplicationManager {
- private final ReplicationQueuesClient replicationQueuesClient;
- private final ReplicationPeers replicationPeers;
-
- public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable
abortable)
- throws IOException {
-try {
- this.replicationQueuesClient = ReplicationFactory
- .getReplicationQueuesClient(new
ReplicationQueuesClientArguments(conf, abortable, zkw));
- this.replicationQueuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf,
-this.replicationQueuesClient, abortable);
- this.replicationPeers.init();
-} catch (Exception e) {
- throw new IOException("Failed to construct ReplicationManager", e);
-}
- }
-
- public void addReplicationPeer(String peerId, ReplicationPeerConfig
peerConfig, boolean enabled)
- throws ReplicationException {
-checkPeerConfig(peerConfig);
-replicationPeers.registerPeer(peerId, peerConfig, enabled);
-replicationPeers.peerConnected(peerId);
- }
-
- public void removeReplicationPeer(String peerId) throws ReplicationException
{
-replicationPeers.peerDisconnected(peerId);
-replicationPeers.unregisterPeer(peerId);
- }
-
- public void enableReplicationPeer(String peerId) throws ReplicationException
{
-this.replicationPeers.enablePeer(peerId);
- }
-
- public void disableReplicationPeer(String peerId) throws
ReplicationException {
-this.replicationPeers.disablePeer(peerId);
- }
-
- public ReplicationPeerConfig getPeerConfig(String peerId)
- throws ReplicationException, ReplicationPeerNotFoundException {
-ReplicationPeerConfig peerConfig =
replicationPeers.getReplicationPeerConfig(peerId);
-if (peerConfig == null) {
- throw new ReplicationPeerNotFoundException(peerId);
-}
-return peerConfig;
- }
-
- public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException, IOException {
-checkPeerConfig(peerConfig);
-this.replicationPeers.updatePeerConfig(peerId, peerConfig);
- }
-
- public List listReplicationPeers(Pattern pattern)
- throws ReplicationException {
-List peers = new ArrayList<>();
-List peerIds = replicationPeers.getAllPeerIds();
-for (String peerId : peer
[21/50] [abbrv] hbase git commit: HBASE-19630 Add peer cluster key check when add new replication peer
HBASE-19630 Add peer cluster key check when add new replication peer
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff40fc52
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff40fc52
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff40fc52
Branch: refs/heads/HBASE-19397-branch-2
Commit: ff40fc52895f9b39ae064b89c90ad3234391b554
Parents: 5bfef7d
Author: Guanghao Zhang
Authored: Tue Dec 26 21:10:00 2017 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:52:50 2018 +0800
--
.../replication/ReplicationPeerManager.java | 54
.../replication/TestReplicationAdmin.java | 22
2 files changed, 54 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff40fc52/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 84abfeb..b78cbce 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,6 +43,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -216,36 +218,36 @@ public final class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
- /**
- * If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
- * Then allow config exclude namespaces or exclude table-cfs which can't be
replicated to peer
- * cluster.
- *
- * If replicate_all flag is false, it means all user tables can't be
replicated to peer cluster.
- * Then allow to config namespaces or table-cfs which will be replicated to
peer cluster.
- */
- private static void checkPeerConfig(ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException {
+ private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws
DoNotRetryIOException {
+checkClusterKey(peerConfig.getClusterKey());
+
if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty()) ||
-(peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
-throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly " +
- "when you want replicate all cluster");
+ // If replicate_all flag is true, it means all user tables will be
replicated to peer cluster.
+ // Then allow config exclude namespaces or exclude table-cfs which can't
be replicated to peer
+ // cluster.
+ if ((peerConfig.getNamespaces() != null &&
!peerConfig.getNamespaces().isEmpty())
+ || (peerConfig.getTableCFsMap() != null &&
!peerConfig.getTableCFsMap().isEmpty())) {
+throw new DoNotRetryIOException("Need clean namespaces or table-cfs
config firstly "
++ "when you want replicate all cluster");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
peerConfig.getExcludeTableCFsMap());
} else {
- if ((peerConfig.getExcludeNamespaces() != null &&
-!peerConfig.getExcludeNamespaces().isEmpty()) ||
-(peerConfig.getExcludeTableCFsMap() != null &&
- !peerConfig.getExcludeTableCFsMap().isEmpty())) {
+ // If replicate_all flag is false, it means all user tables can't be
replicated to peer
+ // cluster. Then allow to config namespaces or table-cfs which will be
replicated to peer
+ // cluster.
+ if ((peerConfig.getExcludeNamespaces() != null
+ && !peerConfig.getExcludeNamespaces().isEmpty())
+ || (peerConfig.getExcludeTableCFsMap() != null
+ && !peerConfig.getExcludeTableCFsMap().isEmpty())) {
throw new DoNotRetryIOException(
-"Need clean exclude-namespaces or exclude-table-cfs config
firstly"
[15/50] [abbrv] hbase git commit: HBASE-20088 Update NOTICE.txt year
HBASE-20088 Update NOTICE.txt year Signed-off-by: Andrew Purtell Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3443aa96 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3443aa96 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3443aa96 Branch: refs/heads/HBASE-19064 Commit: 3443aa96b4357d06181fafd9ad03d6a846df9d88 Parents: e47d1e4 Author: Josh Elser Authored: Mon Feb 26 17:00:50 2018 -0500 Committer: Josh Elser Committed: Tue Feb 27 09:52:30 2018 -0500 -- NOTICE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3443aa96/NOTICE.txt -- diff --git a/NOTICE.txt b/NOTICE.txt index 9c238be..ba4b609 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Apache HBase -Copyright 2007-2017 The Apache Software Foundation +Copyright 2007-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
[01/50] [abbrv] hbase git commit: HBASE-20043 ITBLL fails against hadoop3 [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19064 0ac769246 -> 3301f9757 (forced update)
HBASE-20043 ITBLL fails against hadoop3
Fix MoveRandomRegionOfTableAction. It depended on old AM behavior.
Make it do explicit move as is required in AMv3; w/o it, it was just
closing region causing test to fail.
Fix pom so hadoop3 profile specifies a different netty3 version.
Bunch of logging format change that came of trying trying to read
the spew from this test.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/549a6d93
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/549a6d93
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/549a6d93
Branch: refs/heads/HBASE-19064
Commit: 549a6d93d4a0716f2965c6119b6417f4a3634b12
Parents: ba5fb53
Author: Michael Stack
Authored: Fri Feb 23 14:53:52 2018 -0800
Committer: Michael Stack
Committed: Sat Feb 24 17:29:54 2018 -0800
--
.../apache/hadoop/hbase/io/ByteBufferPool.java | 5 +--
.../hadoop/hbase/chaos/actions/Action.java | 32 +-
.../actions/MoveRandomRegionOfTableAction.java | 13
.../chaos/actions/MoveRegionsOfTableAction.java | 35
.../chaos/monkies/PolicyBasedChaosMonkey.java | 2 +-
.../hbase/chaos/policies/PeriodicPolicy.java| 6 ++--
.../policies/PeriodicRandomActionPolicy.java| 3 +-
.../procedure2/RemoteProcedureDispatcher.java | 4 +--
.../hadoop/hbase/io/hfile/CacheConfig.java | 2 +-
.../hadoop/hbase/io/hfile/LruBlockCache.java| 14
.../apache/hadoop/hbase/ipc/NettyRpcServer.java | 4 +--
.../apache/hadoop/hbase/ipc/RpcExecutor.java| 10 +++---
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../RegionServerProcedureManagerHost.java | 8 ++---
.../hbase/regionserver/CompactingMemStore.java | 5 +--
.../hbase/regionserver/CompactionPipeline.java | 2 +-
.../hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/regionserver/HRegionServer.java | 4 +--
.../hadoop/hbase/regionserver/HStore.java | 2 +-
.../hbase/regionserver/HeapMemoryManager.java | 10 +++---
.../hadoop/hbase/regionserver/Leases.java | 3 +-
.../MemStoreCompactionStrategy.java | 5 ++-
.../hbase/regionserver/MemStoreCompactor.java | 3 +-
.../hadoop/hbase/regionserver/Segment.java | 2 +-
.../compactions/CompactionConfiguration.java| 9 ++---
.../compactions/ExploringCompactionPolicy.java | 6 ++--
.../PressureAwareThroughputController.java | 2 +-
.../replication/regionserver/Replication.java | 2 +-
pom.xml | 9 +++--
29 files changed, 108 insertions(+), 98 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/549a6d93/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
index 6c009b3..caca20b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferPool.java
@@ -80,8 +80,9 @@ public class ByteBufferPool {
this.maxPoolSize = maxPoolSize;
this.directByteBuffer = directByteBuffer;
// TODO can add initialPoolSize config also and make those many BBs ready
for use.
-LOG.info("Created ByteBufferPool with bufferSize : " + bufferSize + " and
maxPoolSize : "
-+ maxPoolSize);
+LOG.info("Created with bufferSize={} and maxPoolSize={}",
+org.apache.hadoop.util.StringUtils.byteDesc(bufferSize),
+org.apache.hadoop.util.StringUtils.byteDesc(maxPoolSize));
this.count = new AtomicInteger(0);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/549a6d93/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
--
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
index ae8cd1f..2b2c1b8 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
@@ -138,63 +138,63 @@ public class Action {
}
protected void killMaster(ServerName server) throws IOException {
-LOG.info("Killing master:" + server);
+LOG.info("Killing master " + server);
cluster.killMaster(server);
cluster.waitForMasterToStop(server, killMasterTimeout);
-LOG.info("Killed master server:" + server);
+LOG.info("Killed master " + server);
}
protected
[34/50] [abbrv] hbase git commit: HBASE-20104 Fix infinite loop of RIT when creating table on a rsgroup that has no online servers
HBASE-20104 Fix infinite loop of RIT when creating table on a rsgroup that has
no online servers
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee1f26c4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee1f26c4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee1f26c4
Branch: refs/heads/HBASE-19064
Commit: ee1f26c4bbb5490e257eee63933fd3943f102a7d
Parents: 776eb5d
Author: haxiaolin
Authored: Thu Mar 1 08:34:31 2018 -0800
Committer: tedyu
Committed: Thu Mar 1 08:34:31 2018 -0800
--
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 33 +++
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 43
2 files changed, 76 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee1f26c4/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index 7fec32d..624aa44 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -31,6 +31,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
@@ -352,6 +353,27 @@ public class RSGroupAdminEndpoint implements
MasterCoprocessor, MasterObserver {
}
}
+ boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException {
+String groupName =
+
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
+.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
+if (groupName == null) {
+ groupName = RSGroupInfo.DEFAULT_GROUP;
+}
+RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
+if (rsGroupInfo == null) {
+ throw new ConstraintException(
+ "Default RSGroup (" + groupName + ") for this table's " + "namespace
does not exist.");
+}
+
+for (ServerName onlineServer :
master.getServerManager().createDestinationServersList()) {
+ if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) {
+return true;
+ }
+}
+return false;
+ }
+
void assignTableToGroup(TableDescriptor desc) throws IOException {
String groupName =
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
@@ -374,6 +396,17 @@ public class RSGroupAdminEndpoint implements
MasterCoprocessor, MasterObserver {
// MasterObserver overrides
/
+ @Override
+ public void preCreateTableAction(
+ final ObserverContext ctx,
+ final TableDescriptor desc,
+ final RegionInfo[] regions) throws IOException {
+if (!desc.getTableName().isSystemTable() &&
!rsgroupHasServersOnline(desc)) {
+ throw new HBaseIOException("No online servers in the rsgroup, which
table " +
+ desc.getTableName().getNameAsString() + " belongs to");
+}
+ }
+
// Assign table to default RSGroup.
@Override
public void postCreateTable(ObserverContext
ctx,
http://git-wip-us.apache.org/repos/asf/hbase/blob/ee1f26c4/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index 76bcd20..fef3a5e 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -999,4 +999,47 @@ public abstract class TestRSGroupsBase {
assertFalse(newGroupServers.contains(targetServer.getAddress()));
assertEquals(2, newGroupServers.size());
}
+
+ @Test
+ public void testCreateWhenRsgroupNoOnlineServers() throws Exception {
+LOG.info("testCreateWhenRsgroupNoOnlineServers");
+
+// set rsgroup has no online servers and test create table
+final RSGroupInfo appInfo = addGroup("appInfo", 1);
+Iterator iterator = appInfo.getServers().iterator();
+List serversToDecommission = new ArrayList<>();
+ServerName targetServer =
ServerName.parseServerName(iterator.next().toString());
+AdminPr
[41/50] [abbrv] hbase git commit: HBASE-19747 Introduce a special WALProvider for synchronous replication
HBASE-19747 Introduce a special WALProvider for synchronous replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a2d39f94
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a2d39f94
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a2d39f94
Branch: refs/heads/HBASE-19064
Commit: a2d39f944e0acc89553c3412d384fbea8959eb3f
Parents: cded0fb
Author: zhangduo
Authored: Fri Jan 19 18:38:39 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:25:13 2018 +0800
--
.../hbase/regionserver/wal/AbstractFSWAL.java | 7 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 1 -
.../hbase/regionserver/wal/DualAsyncFSWAL.java | 4 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 3 -
.../regionserver/PeerActionListener.java| 33 +++
.../SynchronousReplicationPeerProvider.java | 35 +++
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 1 +
.../hadoop/hbase/wal/AsyncFSWALProvider.java| 18 +-
.../hbase/wal/NettyAsyncFSWALConfigHelper.java | 8 +-
.../hbase/wal/RegionGroupingProvider.java | 13 +-
.../wal/SynchronousReplicationWALProvider.java | 225 +++
.../org/apache/hadoop/hbase/wal/WALFactory.java | 37 ++-
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 16 +-
.../regionserver/TestCompactionPolicy.java | 1 +
.../regionserver/TestFailedAppendAndSync.java | 122 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 24 +-
.../TestHRegionWithInMemoryFlush.java | 7 -
.../hbase/regionserver/TestRegionIncrement.java | 20 +-
.../hbase/regionserver/TestWALLockup.java | 1 +
.../regionserver/wal/AbstractTestWALReplay.java | 1 +
.../regionserver/wal/ProtobufLogTestHelper.java | 44 +++-
.../hbase/regionserver/wal/TestAsyncFSWAL.java | 13 +-
.../regionserver/wal/TestAsyncWALReplay.java| 4 +-
.../wal/TestCombinedAsyncWriter.java| 3 +-
.../hbase/regionserver/wal/TestFSHLog.java | 15 +-
.../hbase/regionserver/wal/TestWALReplay.java | 1 +
.../apache/hadoop/hbase/wal/IOTestProvider.java | 2 -
.../TestSynchronousReplicationWALProvider.java | 153 +
28 files changed, 659 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2d39f94/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index ce8dafa..4816d77 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -430,6 +430,13 @@ public abstract class AbstractFSWAL
implements WAL {
this.implClassName = getClass().getSimpleName();
}
+ /**
+ * Used to initialize the WAL. Usually just call rollWriter to create the
first log writer.
+ */
+ public void init() throws IOException {
+rollWriter();
+ }
+
@Override
public void registerWALActionsListener(WALActionsListener listener) {
this.listeners.add(listener);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2d39f94/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 0bee9d6..17133ed 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -248,7 +248,6 @@ public class AsyncFSWAL extends AbstractFSWAL {
batchSize = conf.getLong(WAL_BATCH_SIZE, DEFAULT_WAL_BATCH_SIZE);
waitOnShutdownInSeconds =
conf.getInt(ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS,
DEFAULT_ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS);
-rollWriter();
}
private static boolean waitingRoll(int epochAndState) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a2d39f94/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
index 42b0dae..0495337 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
+++
b/hbase-serv
[40/50] [abbrv] hbase git commit: HBASE-19078 Add a remote peer cluster wal directory config for synchronous replication
HBASE-19078 Add a remote peer cluster wal directory config for synchronous
replication
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cded0fb0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cded0fb0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cded0fb0
Branch: refs/heads/HBASE-19064
Commit: cded0fb0a9e27d72445139bd9d20f67db5d982ad
Parents: fc7fee0
Author: Guanghao Zhang
Authored: Sat Jan 13 18:55:28 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:25:13 2018 +0800
--
.../replication/ReplicationPeerConfigUtil.java | 6 ++
.../replication/ReplicationPeerConfig.java | 21 +-
.../ReplicationPeerConfigBuilder.java | 7 ++
.../src/main/protobuf/Replication.proto | 1 +
.../replication/ReplicationPeerManager.java | 15
.../replication/TestReplicationAdmin.java | 77
.../src/main/ruby/hbase/replication_admin.rb| 17 +++--
hbase-shell/src/main/ruby/hbase_constants.rb| 1 +
.../src/main/ruby/shell/commands/add_peer.rb| 21 +-
.../src/main/ruby/shell/commands/list_peers.rb | 19 -
.../test/ruby/hbase/replication_admin_test.rb | 16
11 files changed, 188 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cded0fb0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index a234a9b..642149b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -315,6 +315,9 @@ public final class ReplicationPeerConfigUtil {
excludeNamespacesList.stream().map(ByteString::toStringUtf8).collect(Collectors.toSet()));
}
+if (peer.hasRemoteWALDir()) {
+ builder.setRemoteWALDir(peer.getRemoteWALDir());
+}
return builder.build();
}
@@ -371,6 +374,9 @@ public final class ReplicationPeerConfigUtil {
}
}
+if (peerConfig.getRemoteWALDir() != null) {
+ builder.setRemoteWALDir(peerConfig.getRemoteWALDir());
+}
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cded0fb0/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index bf8d030..4c10c46 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -46,6 +46,8 @@ public class ReplicationPeerConfig {
private Map> excludeTableCFsMap =
null;
private Set excludeNamespaces = null;
private long bandwidth = 0;
+ // Used by synchronous replication
+ private String remoteWALDir;
private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) {
this.clusterKey = builder.clusterKey;
@@ -64,6 +66,7 @@ public class ReplicationPeerConfig {
builder.excludeNamespaces != null ?
Collections.unmodifiableSet(builder.excludeNamespaces)
: null;
this.bandwidth = builder.bandwidth;
+this.remoteWALDir = builder.remoteWALDir;
}
private Map>
@@ -210,6 +213,10 @@ public class ReplicationPeerConfig {
return this;
}
+ public String getRemoteWALDir() {
+return this.remoteWALDir;
+ }
+
public static ReplicationPeerConfigBuilder newBuilder() {
return new ReplicationPeerConfigBuilderImpl();
}
@@ -223,7 +230,8 @@ public class ReplicationPeerConfig {
.setReplicateAllUserTables(peerConfig.replicateAllUserTables())
.setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap())
.setExcludeNamespaces(peerConfig.getExcludeNamespaces())
-.setBandwidth(peerConfig.getBandwidth());
+.setBandwidth(peerConfig.getBandwidth())
+.setRemoteWALDir(peerConfig.getRemoteWALDir());
return builder;
}
@@ -250,6 +258,8 @@ public class ReplicationPeerConfig {
private long bandwidth = 0;
+private String remoteWALDir = null;
+
@Override
public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) {
this.clusterKey = clusterKey;
@@ -313,
[33/50] [abbrv] hbase git commit: HBASE-20093 (addendum) remove unused import of ServerLoad
HBASE-20093 (addendum) remove unused import of ServerLoad
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/776eb5d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/776eb5d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/776eb5d9
Branch: refs/heads/HBASE-19064
Commit: 776eb5d9cb1658ee512da21b62d94d9258cf8e50
Parents: 0732ef5
Author: Chia-Ping Tsai
Authored: Thu Mar 1 20:44:27 2018 +0800
Committer: Chia-Ping Tsai
Committed: Thu Mar 1 23:47:36 2018 +0800
--
.../org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon| 1 -
.../src/main/java/org/apache/hadoop/hbase/master/HMaster.java | 3 +--
2 files changed, 1 insertion(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/776eb5d9/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
--
diff --git
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 12e0a69b..3091e18 100644
---
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -37,7 +37,6 @@ org.apache.hadoop.hbase.HBaseConfiguration;
org.apache.hadoop.hbase.HConstants;
org.apache.hadoop.hbase.HTableDescriptor;
org.apache.hadoop.hbase.NamespaceDescriptor;
-org.apache.hadoop.hbase.ServerLoad;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.TableName;
org.apache.hadoop.hbase.client.Admin;
http://git-wip-us.apache.org/repos/asf/hbase/blob/776eb5d9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b0dd0b4..c33f555 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -71,7 +71,6 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -1041,7 +1040,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
// The isServerOnline check is opportunistic, correctness is handled
inside
if (!this.serverManager.isServerOnline(sn) &&
- serverManager.checkAndRecordNewServer(sn, new
ServerLoad(ServerMetricsBuilder.of(sn {
+ serverManager.checkAndRecordNewServer(sn,
ServerMetricsBuilder.of(sn))) {
LOG.info("Registered server found up in zk but who has not yet
reported in: " + sn);
}
}
[43/50] [abbrv] hbase git commit: HBASE-19781 Add a new cluster state flag for synchronous replication
HBASE-19781 Add a new cluster state flag for synchronous replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c7d22055
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c7d22055
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c7d22055
Branch: refs/heads/HBASE-19064
Commit: c7d22055877ed6075fec8ace511c567a55e0dd08
Parents: a2d39f9
Author: Guanghao Zhang
Authored: Mon Jan 22 11:44:49 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../org/apache/hadoop/hbase/client/Admin.java | 39 +
.../apache/hadoop/hbase/client/AsyncAdmin.java | 31
.../hadoop/hbase/client/AsyncHBaseAdmin.java| 7 +
.../hbase/client/ConnectionImplementation.java | 9 ++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 26 +++
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 15 ++
.../client/ShortCircuitMasterConnection.java| 9 ++
.../replication/ReplicationPeerConfigUtil.java | 26 +--
.../replication/ReplicationPeerDescription.java | 10 +-
.../hbase/replication/SyncReplicationState.java | 48 ++
.../hbase/shaded/protobuf/RequestConverter.java | 10 ++
.../src/main/protobuf/Master.proto | 4 +
.../src/main/protobuf/MasterProcedure.proto | 4 +
.../src/main/protobuf/Replication.proto | 20 +++
.../replication/ReplicationPeerStorage.java | 18 ++-
.../hbase/replication/ReplicationUtils.java | 1 +
.../replication/ZKReplicationPeerStorage.java | 60 +--
.../replication/TestReplicationStateBasic.java | 23 ++-
.../TestZKReplicationPeerStorage.java | 12 +-
.../hbase/coprocessor/MasterObserver.java | 23 +++
.../org/apache/hadoop/hbase/master/HMaster.java | 12 ++
.../hbase/master/MasterCoprocessorHost.java | 21 +++
.../hadoop/hbase/master/MasterRpcServices.java | 17 ++
.../hadoop/hbase/master/MasterServices.java | 9 ++
.../procedure/PeerProcedureInterface.java | 2 +-
.../replication/ReplicationPeerManager.java | 51 +-
...ransitPeerSyncReplicationStateProcedure.java | 159 +++
.../hbase/security/access/AccessController.java | 8 +
.../replication/TestReplicationAdmin.java | 62
.../hbase/master/MockNoopMasterServices.java| 11 +-
.../cleaner/TestReplicationHFileCleaner.java| 4 +-
.../TestReplicationTrackerZKImpl.java | 6 +-
.../TestReplicationSourceManager.java | 3 +-
.../security/access/TestAccessController.java | 16 ++
.../hbase/util/TestHBaseFsckReplication.java| 5 +-
.../src/main/ruby/hbase/replication_admin.rb| 15 ++
hbase-shell/src/main/ruby/shell.rb | 1 +
.../src/main/ruby/shell/commands/list_peers.rb | 6 +-
.../transit_peer_sync_replication_state.rb | 44 +
.../test/ruby/hbase/replication_admin_test.rb | 24 +++
40 files changed, 817 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7d22055/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index b8546fa..167d6f3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -52,6 +52,7 @@ import
org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
@@ -2648,6 +2649,44 @@ public interface Admin extends Abortable, Closeable {
List listReplicationPeers(Pattern pattern)
throws IOException;
/**
+ * Transit current cluster to a new state in a synchronous replication peer.
+ * @param peerId a short name that identifies the peer
+ * @param state a new state of current cluster
+ * @throws IOException if a remote or network exception occurs
+ */
+ void transitReplicationPeerSyncReplicationState(String peerId,
SyncReplicationState state)
+ throws IOException;
+
+ /**
+ * Transit current cluster to a new state in a synchronous replication peer.
But does not block
+ * and wait for it.
+ *
+ * You can use Future.get(long, TimeUnit) to wait on the operation to
complete. It may throw
+ * Execu
[10/50] [abbrv] hbase git commit: HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
HBASE-20086 PE randomSeekScan fails with ClassNotFoundException
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3aefe78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3aefe78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3aefe78
Branch: refs/heads/HBASE-19064
Commit: d3aefe783476e860e7b1c474b50cf18a7ae0be00
Parents: b11e506
Author: tedyu
Authored: Mon Feb 26 18:29:35 2018 -0800
Committer: tedyu
Committed: Mon Feb 26 18:29:35 2018 -0800
--
.../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3aefe78/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a5fa2c7..5a63ef4 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -527,7 +527,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
Histogram.class, // yammer metrics
- ObjectMapper.class); // jackson-mapper-asl
+ ObjectMapper.class, // jackson-mapper-asl
+ FilterAllFilter.class // hbase-server tests jar
+ );
TableMapReduceUtil.initCredentials(job);
[38/50] [abbrv] hbase git commit: HBASE-20075 remove logic for branch-1.1 nightly testing
HBASE-20075 remove logic for branch-1.1 nightly testing
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c415ddb3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c415ddb3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c415ddb3
Branch: refs/heads/HBASE-19064
Commit: c415ddb38c26877a76ba4e9b8e46cbb79b7f0574
Parents: 383b7cc
Author: Sean Busbey
Authored: Sat Feb 24 15:50:12 2018 -0600
Committer: Sean Busbey
Committed: Thu Mar 1 15:28:30 2018 -0600
--
dev-support/Jenkinsfile | 9 -
1 file changed, 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c415ddb3/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index fb38bbe..fa7c8ba 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -195,10 +195,6 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
"${BASEDIR}/dev-support/gather_machine_environment.sh"
"${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
- # for branch-1.1 we don't do jdk8 findbugs, so do it here
- if [ "${BRANCH_NAME}" == "branch-1.1" ]; then
-TESTS+=",findbugs"
- fi
rm -rf "${OUTPUT_DIR}/commentfile}"
declare -i status=0
if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
@@ -249,11 +245,6 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
label 'Hadoop'
}
}
- when {
-not {
- branch 'branch-1.1*'
-}
- }
environment {
TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
[09/50] [abbrv] hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server
HBASE-20069 fix existing findbugs errors in hbase-server
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b11e5066
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b11e5066
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b11e5066
Branch: refs/heads/HBASE-19064
Commit: b11e506664614c243c08949c256430d4dd13ba6c
Parents: 73028d5
Author: Michael Stack
Authored: Sat Feb 24 13:01:02 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 16:01:31 2018 -0800
--
.../hbase/io/encoding/EncodedDataBlock.java | 19 +--
.../apache/hadoop/hbase/nio/MultiByteBuff.java | 4 +-
.../hadoop/hbase/nio/TestMultiByteBuff.java | 19 +++
.../hbase/procedure2/ProcedureExecutor.java | 1 -
.../hbase/procedure2/StateMachineProcedure.java | 1 -
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 3 +
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../master/assignment/AssignmentManager.java| 1 -
.../assignment/SplitTableRegionProcedure.java | 7 +-
.../hbase/master/cleaner/CleanerChore.java | 39 ---
.../hadoop/hbase/regionserver/HRegion.java | 3 +-
.../hbase/regionserver/MemStoreFlusher.java | 115 +--
.../hbase/regionserver/RSRpcServices.java | 1 -
.../regionserver/RegionCoprocessorHost.java | 2 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 4 +-
.../hbase/util/compaction/MajorCompactor.java | 9 +-
16 files changed, 147 insertions(+), 82 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
index a791c09..af68656 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
@@ -228,6 +228,7 @@ public class EncodedDataBlock {
*/
public byte[] encodeData() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
+byte [] baosBytes = null;
try {
baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER);
DataOutputStream out = new DataOutputStream(baos);
@@ -255,25 +256,17 @@ public class EncodedDataBlock {
kv.setSequenceId(memstoreTS);
this.dataBlockEncoder.encode(kv, encodingCtx, out);
}
- BufferGrabbingByteArrayOutputStream stream = new
BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- this.dataBlockEncoder.endBlockEncoding(encodingCtx, out,
stream.ourBytes);
+ // Below depends on BAOS internal behavior. toByteArray makes a copy of
bytes so far.
+ baos.flush();
+ baosBytes = baos.toByteArray();
+ this.dataBlockEncoder.endBlockEncoding(encodingCtx, out, baosBytes);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Bug in encoding part of algorithm %s. " +
"Probably it requested more bytes than are available.",
toString()), e);
}
-return baos.toByteArray();
- }
-
- private static class BufferGrabbingByteArrayOutputStream extends
ByteArrayOutputStream {
-private byte[] ourBytes;
-
-@Override
-public synchronized void write(byte[] b, int off, int len) {
- this.ourBytes = b;
-}
+return baosBytes;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b11e5066/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
index fecf012..847e2eb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
@@ -282,7 +282,7 @@ public class MultiByteBuff extends ByteBuff {
return ByteBufferUtils.toShort(item, offsetInItem);
}
if (items.length - 1 == itemIndex) {
- // means cur item is the last one and we wont be able to read a int.
Throw exception
+ // means cur item is the last one and we wont be able to read a short.
Throw exception
throw new BufferUnderflowException();
}
ByteBuffer nextItem = items[itemIndex + 1];
@@ -294,7 +294,7 @@ public class MultiByteBuff extends ByteBuff {
}
for (int i = 0; i < Bytes.SIZEOF_SHORT - remainingLen; i++) {
l = (short) (l << 8);
- l = (short) (l ^ (ByteBuf
[07/50] [abbrv] hbase git commit: HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
HBASE-20036 TestAvoidCellReferencesIntoShippedBlocks timed out (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cfb4643
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cfb4643
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cfb4643
Branch: refs/heads/HBASE-19064
Commit: 7cfb46432fbdf9b53592be11efc8a7d79d1a9455
Parents: a29b3ca
Author: Vasudevan
Authored: Mon Feb 26 22:07:42 2018 +0530
Committer: Vasudevan
Committed: Mon Feb 26 22:07:42 2018 +0530
--
...estAvoidCellReferencesIntoShippedBlocks.java | 30 +++-
1 file changed, 17 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cfb4643/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index 0e12ad6..d22772a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -400,23 +400,27 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
scanner = table.getScanner(s1);
int count = Iterables.size(scanner);
assertEquals("Count the rows", 2, count);
- iterator = cache.iterator();
- List newCacheList = new ArrayList<>();
- while (iterator.hasNext()) {
-CachedBlock next = iterator.next();
-BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(),
next.getOffset());
-newCacheList.add(cacheKey);
- }
int newBlockRefCount = 0;
- for (BlockCacheKey key : cacheList) {
-if (newCacheList.contains(key)) {
- newBlockRefCount++;
+ List newCacheList = new ArrayList<>();
+ while (true) {
+newBlockRefCount = 0;
+newCacheList.clear();
+iterator = cache.iterator();
+while (iterator.hasNext()) {
+ CachedBlock next = iterator.next();
+ BlockCacheKey cacheKey = new
BlockCacheKey(next.getFilename(), next.getOffset());
+ newCacheList.add(cacheKey);
+}
+for (BlockCacheKey key : cacheList) {
+ if (newCacheList.contains(key)) {
+newBlockRefCount++;
+ }
+}
+if (newBlockRefCount == 6) {
+ break;
}
}
-
- assertEquals("old blocks should still be found ", 6,
newBlockRefCount);
latch.countDown();
-
} catch (IOException e) {
}
}
[47/50] [abbrv] hbase git commit: HBASE-19935 Only allow table replication for sync replication for now
HBASE-19935 Only allow table replication for sync replication for now
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c825551f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c825551f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c825551f
Branch: refs/heads/HBASE-19064
Commit: c825551f4176a0b70db5119fff00ee70dbf00ac8
Parents: c6a2c9d
Author: Guanghao Zhang
Authored: Tue Feb 6 16:00:59 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../replication/ReplicationPeerConfig.java | 9 +++
.../replication/ReplicationPeerManager.java | 34 -
.../replication/TestReplicationAdmin.java | 73 ++--
.../wal/TestCombinedAsyncWriter.java| 6 ++
.../wal/TestSyncReplicationWALProvider.java | 6 ++
5 files changed, 102 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c825551f/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index 4c10c46..69565a7 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
@@ -217,6 +219,13 @@ public class ReplicationPeerConfig {
return this.remoteWALDir;
}
+ /**
+ * Use remote wal dir to decide whether a peer is sync replication peer
+ */
+ public boolean isSyncReplication() {
+return !StringUtils.isBlank(this.remoteWALDir);
+ }
+
public static ReplicationPeerConfigBuilder newBuilder() {
return new ReplicationPeerConfigBuilderImpl();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c825551f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 9336fbd..6bfd9c9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -167,7 +167,7 @@ public class ReplicationPeerManager {
" does not match new remote wal dir '" +
peerConfig.getRemoteWALDir() + "'");
}
-if (oldPeerConfig.getRemoteWALDir() != null) {
+if (oldPeerConfig.isSyncReplication()) {
if (!ReplicationUtils.isKeyConfigEqual(oldPeerConfig, peerConfig)) {
throw new DoNotRetryIOException(
"Changing the replicated namespace/table config on a synchronous
replication "
@@ -195,8 +195,8 @@ public class ReplicationPeerManager {
}
ReplicationPeerConfig copiedPeerConfig =
ReplicationPeerConfig.newBuilder(peerConfig).build();
SyncReplicationState syncReplicationState =
-StringUtils.isBlank(peerConfig.getRemoteWALDir()) ?
SyncReplicationState.NONE
-: SyncReplicationState.DOWNGRADE_ACTIVE;
+copiedPeerConfig.isSyncReplication() ?
SyncReplicationState.DOWNGRADE_ACTIVE
+: SyncReplicationState.NONE;
peerStorage.addPeer(peerId, copiedPeerConfig, enabled,
syncReplicationState);
peers.put(peerId,
new ReplicationPeerDescription(peerId, enabled, copiedPeerConfig,
syncReplicationState));
@@ -316,9 +316,37 @@ public class ReplicationPeerManager {
peerConfig.getTableCFsMap());
}
+if (peerConfig.isSyncReplication()) {
+ checkPeerConfigForSyncReplication(peerConfig);
+}
+
checkConfiguredWALEntryFilters(peerConfig);
}
+ private void checkPeerConfigForSyncReplication(ReplicationPeerConfig
peerConfig)
+ throws DoNotRetryIOException {
+// This is used to reduce the difficulty for implementing the sync
replication state transition
+// as we need to reopen all the related regions.
+// TODO: Add namespace, replicat_all flag back
+if (peerConfig.replicateAllUserTables()) {
+ throw new DoNotRetryIOException(
+ "Only support replicated table config for sync
[18/50] [abbrv] hbase git commit: HBASE-19614 Use ArrayDeque as Queue instead of LinkedList in CompoundBloomFilterWriter
HBASE-19614 Use ArrayDeque as Queue instead of LinkedList in CompoundBloomFilterWriter Signed-off-by: Apekshit Sharma Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f01c90bc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f01c90bc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f01c90bc Branch: refs/heads/HBASE-19064 Commit: f01c90bc0e3b44fbc2c500a631cd49202d2cb975 Parents: 0fa5d69 Author: BELUGA BEHR Authored: Tue Feb 27 11:49:30 2018 -0800 Committer: Apekshit Sharma Committed: Tue Feb 27 11:50:52 2018 -0800 -- .../apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/f01c90bc/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java index 0b58b21..d63a824 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.io.hfile; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.LinkedList; +import java.util.ArrayDeque; import java.util.Queue; import org.apache.hadoop.hbase.Cell; @@ -71,7 +71,7 @@ public class CompoundBloomFilterWriter extends CompoundBloomFilterBase BloomFilterChunk chunk; } - private Queue readyChunks = new LinkedList<>(); + private Queue readyChunks = new ArrayDeque<>(); /** The first key in the current Bloom filter chunk. */ private byte[] firstKeyInChunk = null;
[20/50] [abbrv] hbase git commit: HBASE-20093 Replace ServerLoad by ServerMetrics for ServerManager
HBASE-20093 Replace ServerLoad by ServerMetrics for ServerManager
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7f6e971c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7f6e971c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7f6e971c
Branch: refs/heads/HBASE-19064
Commit: 7f6e971c4cc2e3906f959c6304fc05faa7703054
Parents: ba063ab
Author: Chia-Ping Tsai
Authored: Tue Feb 27 23:20:06 2018 +0800
Committer: Chia-Ping Tsai
Committed: Wed Feb 28 14:57:50 2018 +0800
--
.../hbase/coprocessor/TestClassLoading.java | 36 ++-
.../hbase/rsgroup/TestRSGroupsOfflineMode.java | 4 +-
.../hbase/tmpl/master/RSGroupListTmpl.jamon | 83 ++---
.../tmpl/master/RegionServerListTmpl.jamon | 99 --
.../hbase/favored/FavoredNodeLoadBalancer.java | 16 +-
.../hadoop/hbase/master/MasterDumpServlet.java | 10 +-
.../hadoop/hbase/master/MasterRpcServices.java | 12 +-
.../hadoop/hbase/master/ServerManager.java | 47 +--
.../hbase/master/balancer/BaseLoadBalancer.java | 10 +-
.../balancer/FavoredStochasticBalancer.java | 10 +-
.../normalizer/SimpleRegionNormalizer.java | 13 +-
.../resources/hbase-webapps/master/rsgroup.jsp | 89 --
.../resources/hbase-webapps/master/table.jsp| 300 ---
.../normalizer/TestSimpleRegionNormalizer.java | 12 +-
14 files changed, 381 insertions(+), 360 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f6e971c/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
--
diff --git
a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
index 922977c..bc75881 100644
---
a/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
+++
b/hbase-endpoint/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
@@ -22,8 +22,14 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-import java.io.*;
-import java.util.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,8 +40,8 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.RegionMetrics;
+import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@@ -47,8 +53,10 @@ import
org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.*;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -482,13 +490,13 @@ public class TestClassLoading {
* @param tableName : given table.
* @return subset of all servers.
*/
- Map serversForTable(String tableName) {
-Map serverLoadHashMap = new HashMap<>();
-for(Map.Entry server:
+ Map serversForTable(String tableName) {
+Map serverLoadHashMap = new HashMap<>();
+for(Map.Entry server:
TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
getOnlineServers().entrySet()) {
- for( Map.Entry region:
- server.getValue().getRegionsLoad().entrySet()) {
+ for(Map.Entry region:
+ server.getValue().getRegionMetrics().entrySet()) {
if (region.getValue().getNameAsString().equals(tableName)) {
// this server hosts a region of tableName: add this server..
serverLoadHashMap.put(server.getKey(),server.getValue());
@@ -501,8 +509,7 @@ public class TestClassLoading {
}
void assertAllRegionServers(String tableName) throws InterruptedException {
-Map servers;
-String[] actualCoprocessors = null;
+Map servers;
boolean success = false;
String[] expectedCoprocessors = regionServerSystemCoprocessors;
[03/50] [abbrv] hbase git commit: HBASE-20019 Document the ColumnValueFilter
HBASE-20019 Document the ColumnValueFilter
Signed-off-by: Chia-Ping Tsai
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a34f129a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a34f129a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a34f129a
Branch: refs/heads/HBASE-19064
Commit: a34f129a9b5e1098f1c9e86b1b8e7202bb97
Parents: a8471bd
Author: Reid Chan
Authored: Mon Feb 26 11:31:08 2018 +0800
Committer: Chia-Ping Tsai
Committed: Mon Feb 26 14:59:42 2018 +0800
--
src/main/asciidoc/_chapters/architecture.adoc | 35 ++
1 file changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a34f129a/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc
b/src/main/asciidoc/_chapters/architecture.adoc
index 9091d5e..6fb5891 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -321,6 +321,41 @@ SingleColumnValueFilter filter = new
SingleColumnValueFilter(
scan.setFilter(filter);
+[[client.filter.cv.cvf]]
+ ColumnValueFilter
+
+Introduced in HBase-2.0.0 version as a complementation of
SingleColumnValueFilter, ColumnValueFilter
+gets matched cell only, while SingleColumnValueFilter gets the entire row
+(has other columns and values) to which the matched cell belongs. Parameters
of constructor of
+ColumnValueFilter are the same as SingleColumnValueFilter.
+[source,java]
+
+ColumnValueFilter filter = new ColumnValueFilter(
+ cf,
+ column,
+ CompareOperaor.EQUAL,
+ Bytes.toBytes("my value")
+ );
+scan.setFilter(filter);
+
+
+Note. For simple query like "equals to a family:qualifier:value", we highly
recommend to use the
+following way instead of using SingleColumnValueFilter or ColumnValueFilter:
+[source,java]
+
+Scan scan = new Scan();
+scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qualifier"));
+ValueFilter vf = new ValueFilter(CompareOperator.EQUAL,
+ new BinaryComparator(Bytes.toBytes("value")));
+scan.setFilter(vf);
+...
+
+This scan will restrict to the specified column 'family:qualifier', avoiding
scan unrelated
+families and columns, which has better performance, and `ValueFilter` is the
condition used to do
+the value filtering.
+
+But if query is much more complicated beyond this book, then please make your
good choice case by case.
+
[[client.filter.cvp]]
=== Column Value Comparators
[36/50] [abbrv] hbase git commit: HBASE-18467 report nightly results to devs via jira
HBASE-18467 report nightly results to devs via jira
- rely on parallel pipeline to ensure all stages always run
- define non-CPS jira commenting function
- comment on jiras in the changeset with summary and links
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ddc9af90
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ddc9af90
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ddc9af90
Branch: refs/heads/HBASE-19064
Commit: ddc9af9027d026aaef94fc8f37c7dc58eaa19d3d
Parents: e86ffed
Author: Sean Busbey
Authored: Wed Aug 9 00:48:46 2017 -0500
Committer: Sean Busbey
Committed: Thu Mar 1 13:11:59 2018 -0600
--
dev-support/Jenkinsfile | 592 ++
dev-support/hbase_nightly_source-artifact.sh | 1 -
2 files changed, 367 insertions(+), 226 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc9af90/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 201783b..fe3676d 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -15,11 +15,7 @@
// specific language governing permissions and limitations
// under the License.
pipeline {
- agent {
-node {
- label 'Hadoop'
-}
- }
+ agent any
triggers {
cron('@daily')
}
@@ -34,6 +30,12 @@ pipeline {
// where we check out to across stages
BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
+// where we'll write everything from different steps. Need a copy here so
the final step can check for success/failure.
+OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
+OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
+OUTPUT_DIR_RELATIVE_HADOOP2 = 'output-jdk8-hadoop2'
+OUTPUT_DIR_RELATIVE_HADOOP3 = 'output-jdk8-hadoop3'
+
PROJECT = 'hbase'
PROJECT_PERSONALITY =
'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older
branches have our old version of the check for said tag.
@@ -62,6 +64,7 @@ pipeline {
dir('component') {
checkout scm
}
+stash name: 'component', includes: "component/*,component/**/*"
}
}
stage ('yetus install') {
@@ -111,252 +114,391 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
stash name: 'yetus', includes:
"yetus-*/*,yetus-*/**/*,tools/personality.sh"
}
}
-stage ('yetus general check') {
- environment {
-// TODO does hadoopcheck need to be jdk specific?
-// Should be things that work with multijdk
-TESTS = 'all,-unit,-findbugs'
-// on branches that don't support jdk7, this will already be
JAVA_HOME, so we'll end up not
-// doing multijdk there.
-MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
-OUTPUT_DIR_RELATIVE = "output-general"
-OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
- }
- steps {
-unstash 'yetus'
-sh '''#!/usr/bin/env bash
- rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
- rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh"
"${OUTPUT_DIR_RELATIVE}/machine"
+stage ('health checks') {
+ parallel {
+stage ('yetus general check') {
+ agent {
+node {
+ label 'Hadoop'
+}
+ }
+ environment {
+// TODO does hadoopcheck need to be jdk specific?
+// Should be things that work with multijdk
+TESTS = 'all,-unit,-findbugs'
+// on branches that don't support jdk7, this will already be
JAVA_HOME, so we'll end up not
+// doing multijdk there.
+MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
+OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
+OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
+ }
+ steps {
+unstash 'yetus'
+unstash 'component'
+sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh"
"${OUTPUT_DIR_RELATIVE}/machine"
'''
-// TODO should this be a download from master, similar to how the
personality is?
-sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
- }
- post {
-always {
- // Has to be relative to WORKSPACE.
- archive "${env.OUTPUT_DIR_RELATIVE}/*"
- archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
[49/50] [abbrv] hbase git commit: HBASE-19990 Create remote wal directory when transitting to state S
HBASE-19990 Create remote wal directory when transitting to state S
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3301f975
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3301f975
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3301f975
Branch: refs/heads/HBASE-19064
Commit: 3301f9757c1926ccb0113a531b9b531b6f478e73
Parents: 010cc7d
Author: zhangduo
Authored: Wed Feb 14 16:01:16 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../procedure2/ProcedureYieldException.java | 9 --
.../hbase/replication/ReplicationUtils.java | 2 ++
.../hadoop/hbase/master/MasterFileSystem.java | 23
.../master/procedure/MasterProcedureEnv.java| 5
...ransitPeerSyncReplicationStateProcedure.java | 29
.../hbase/replication/TestSyncReplication.java | 8 ++
6 files changed, 58 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3301f975/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
index 0487ac5b..dbb9981 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
@@ -15,16 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.procedure2;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
-// TODO: Not used yet
+/**
+ * Indicate that a procedure wants to be rescheduled. Usually because there
are something wrong but
+ * we do not want to fail the procedure.
+ *
+ * TODO: need to support scheduling after a delay.
+ */
@InterfaceAudience.Private
@InterfaceStability.Stable
public class ProcedureYieldException extends ProcedureException {
+
/** default constructor */
public ProcedureYieldException() {
super();
http://git-wip-us.apache.org/repos/asf/hbase/blob/3301f975/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 9d0c05d..9699e7b 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -41,6 +41,8 @@ public final class ReplicationUtils {
public static final String REPLICATION_ATTR_NAME = "__rep__";
+ public static final String REMOTE_WAL_DIR_NAME = "remoteWALs";
+
private ReplicationUtils() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3301f975/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index a37fd4e..9bb8858 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -133,7 +134,6 @@ public class MasterFileSystem {
* Idempotent.
*/
private void createInitialFileSystemLayout() throws IOException {
-
final String[] protectedSubDirs = new String[] {
HConstants.BASE_NAMESPACE_DIR,
HConstants.HFILE_ARCHIVE_DIRECTORY,
@@ -145,7 +145,8 @@ public class MasterFileSystem {
HConstants.HREGION_LOGDIR_NAME,
HConstants.HREGION_OLDLOGDIR_NAME,
HConstants.CORRUPT_DIR_NAME,
- WALProcedureStore.MASTER_PROCEDURE_LOGDIR
+ WALProcedureStore.MASTER_PROCEDURE_LOGDIR,
+
[22/50] [abbrv] hbase git commit: HBASE-20097 Merge TableDescriptors#getAll and TableDescriptors#getAllDescriptors into one
HBASE-20097 Merge TableDescriptors#getAll and
TableDescriptors#getAllDescriptors into one
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/62ee7d95
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/62ee7d95
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/62ee7d95
Branch: refs/heads/HBASE-19064
Commit: 62ee7d9502d599f704ea8c9cf16f9ac4c1b1d22a
Parents: 197bd79
Author: Chia-Ping Tsai
Authored: Tue Feb 27 16:34:14 2018 +0800
Committer: Chia-Ping Tsai
Committed: Wed Feb 28 15:21:21 2018 +0800
--
.../org/apache/hadoop/hbase/TableDescriptors.java | 9 -
.../hadoop/hbase/master/TableStateManager.java | 2 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 16 +---
.../hbase/master/assignment/MockMasterServices.java | 5 -
4 files changed, 2 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/62ee7d95/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
index ecdfc82..5787f66 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java
@@ -55,15 +55,6 @@ public interface TableDescriptors {
throws IOException;
/**
- * Get Map of all TableDescriptors. Populates the descriptor cache as a
- * side effect.
- * @return Map of all descriptors.
- * @throws IOException
- */
- Map getAllDescriptors()
- throws IOException;
-
- /**
* Add or update descriptor
* @param htd Descriptor to set into TableDescriptors
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/62ee7d95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
index 0d89eef..affb684 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java
@@ -223,7 +223,7 @@ public class TableStateManager {
private void fixTableStates(TableDescriptors tableDescriptors, Connection
connection)
throws IOException {
-final Map allDescriptors =
tableDescriptors.getAllDescriptors();
+final Map allDescriptors =
tableDescriptors.getAll();
final Map states = new HashMap<>();
// NOTE: Ful hbase:meta table scan!
MetaTableAccessor.fullScanTables(connection, new
MetaTableAccessor.Visitor() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/62ee7d95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index c72b9e0..b4b0be0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -248,7 +248,7 @@ public class FSTableDescriptors implements TableDescriptors
{
* Returns a map from table name to table descriptor for all tables.
*/
@Override
- public Map getAllDescriptors()
+ public Map getAll()
throws IOException {
Map tds = new TreeMap<>();
@@ -282,20 +282,6 @@ public class FSTableDescriptors implements
TableDescriptors {
}
/**
- * Returns a map from table name to table descriptor for all tables.
- */
- @Override
- public Map getAll() throws IOException {
-Map htds = new TreeMap<>();
-Map allDescriptors = getAllDescriptors();
-for (Map.Entry entry : allDescriptors
-.entrySet()) {
- htds.put(entry.getKey(), entry.getValue());
-}
-return htds;
- }
-
- /**
* Find descriptors by namespace.
* @see #get(org.apache.hadoop.hbase.TableName)
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/62ee7d95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.ja
[05/50] [abbrv] hbase git commit: HBASE-20083 Fix findbugs error for ReplicationSyncUp
HBASE-20083 Fix findbugs error for ReplicationSyncUp
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2beda62a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2beda62a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2beda62a
Branch: refs/heads/HBASE-19064
Commit: 2beda62a10f0828eb10cec28b0ba53246cd0b671
Parents: 8c74d17
Author: zhangduo
Authored: Mon Feb 26 16:37:58 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 22:13:13 2018 +0800
--
.../regionserver/ReplicationSyncUp.java | 59 ++--
.../replication/TestReplicationSyncUpTool.java | 6 +-
2 files changed, 19 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2beda62a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 283eb96..c2862de 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -49,35 +49,18 @@ import org.apache.hadoop.util.ToolRunner;
*/
public class ReplicationSyncUp extends Configured implements Tool {
- private static Configuration conf;
-
private static final long SLEEP_TIME = 1;
- // although the tool is designed to be run on command line
- // this api is provided for executing the tool through another app
- public static void setConfigure(Configuration config) {
-conf = config;
- }
-
/**
* Main program
- * @param args
- * @throws Exception
*/
public static void main(String[] args) throws Exception {
-if (conf == null) conf = HBaseConfiguration.create();
-int ret = ToolRunner.run(conf, new ReplicationSyncUp(), args);
+int ret = ToolRunner.run(HBaseConfiguration.create(), new
ReplicationSyncUp(), args);
System.exit(ret);
}
@Override
public int run(String[] args) throws Exception {
-Replication replication;
-ReplicationSourceManager manager;
-FileSystem fs;
-Path oldLogDir, logDir, walRootDir;
-ZKWatcher zkw;
-
Abortable abortable = new Abortable() {
@Override
public void abort(String why, Throwable e) {
@@ -88,23 +71,19 @@ public class ReplicationSyncUp extends Configured
implements Tool {
return false;
}
};
-
-zkw =
-new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable,
-true);
-
-walRootDir = FSUtils.getWALRootDir(conf);
-fs = FSUtils.getWALFileSystem(conf);
-oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
-logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
-
-System.out.println("Start Replication Server start");
-replication = new Replication();
-replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, null);
-manager = replication.getReplicationManager();
-manager.init().get();
-
-try {
+Configuration conf = getConf();
+try (ZKWatcher zkw =
+ new ZKWatcher(conf, "syncupReplication" + System.currentTimeMillis(),
abortable, true)) {
+ Path walRootDir = FSUtils.getWALRootDir(conf);
+ FileSystem fs = FSUtils.getWALFileSystem(conf);
+ Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
+
+ System.out.println("Start Replication Server start");
+ Replication replication = new Replication();
+ replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir,
null);
+ ReplicationSourceManager manager = replication.getReplicationManager();
+ manager.init().get();
while (manager.activeFailoverTaskCount() > 0) {
Thread.sleep(SLEEP_TIME);
}
@@ -114,15 +93,12 @@ public class ReplicationSyncUp extends Configured
implements Tool {
manager.join();
} catch (InterruptedException e) {
System.err.println("didn't wait long enough:" + e);
- return (-1);
-} finally {
- zkw.close();
+ return -1;
}
-
return 0;
}
- static class DummyServer implements Server {
+ class DummyServer implements Server {
String hostname;
ZKWatcher zkw;
@@ -138,7 +114,7 @@ public class ReplicationSyncUp extends Configured
implements Tool {
@Override
public Configuration getConfiguration() {
- return conf;
+ return getConf();
}
@Override
@@ -191,7 +167,6 @@ public
[16/50] [abbrv] hbase git commit: Revert "for creating patch HBASE-20074-V01.patch"
Revert "for creating patch HBASE-20074-V01.patch"
This reverts commit 73028d5bd9f85655b284654579ddcbbca31e41e8.
bad commit message
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e4ce38df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e4ce38df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e4ce38df
Branch: refs/heads/HBASE-19064
Commit: e4ce38df326e750015f6077b8ca8de9788cce4d4
Parents: 3443aa9
Author: Sean Busbey
Authored: Tue Feb 27 11:20:54 2018 -0600
Committer: Sean Busbey
Committed: Tue Feb 27 11:20:54 2018 -0600
--
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4ce38df/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index d60b049..44b40eb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -136,10 +136,14 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
-// multiply by a factor (the same factor for all index types)
-factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+// multiply by a factor (different factors for different index types)
+if (indexType == IndexType.ARRAY_MAP) {
+ factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-
+} else {
+ factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+ IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
+}
inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to {} and immutable
segments index to type={}",
StringUtils.byteDesc(inmemoryFlushSize), indexType);
[46/50] [abbrv] hbase git commit: HBASE-19857 Complete the procedure for adding a sync replication peer
HBASE-19857 Complete the procedure for adding a sync replication peer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1ee86b9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1ee86b9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1ee86b9
Branch: refs/heads/HBASE-19064
Commit: f1ee86b937d108305ba968eca3bb47ae8e564123
Parents: c7d2205
Author: zhangduo
Authored: Thu Jan 25 20:09:00 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../hbase/replication/ReplicationPeer.java | 9 +
.../hbase/replication/ReplicationPeerImpl.java | 28 +--
.../hbase/replication/ReplicationPeers.java | 3 +-
.../regionserver/PeerActionListener.java| 10 +-
.../SyncReplicationPeerProvider.java| 35 +++
.../SynchronousReplicationPeerProvider.java | 35 ---
.../hbase/wal/SyncReplicationWALProvider.java | 234 +++
.../wal/SynchronousReplicationWALProvider.java | 225 --
.../org/apache/hadoop/hbase/wal/WALFactory.java | 8 +-
.../TestReplicationSourceManager.java | 3 +
.../wal/TestSyncReplicationWALProvider.java | 153
.../TestSynchronousReplicationWALProvider.java | 153
12 files changed, 456 insertions(+), 440 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1ee86b9/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 2da3cce..0196a9a 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,15 @@ public interface ReplicationPeer {
PeerState getPeerState();
/**
+ * Returns the sync replication state of the peer by reading local cache.
+ *
+ * If the peer is not a synchronous replication peer, a {@link
SyncReplicationState#NONE} will be
+ * returned.
+ * @return the sync replication state
+ */
+ SyncReplicationState getSyncReplicationState();
+
+ /**
* Test whether the peer is enabled.
* @return {@code true} if enabled, otherwise {@code false}.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1ee86b9/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index d656466..ff3f662 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -36,6 +36,8 @@ public class ReplicationPeerImpl implements ReplicationPeer {
private volatile PeerState peerState;
+ private volatile SyncReplicationState syncReplicationState;
+
private final List peerConfigListeners;
/**
@@ -45,12 +47,13 @@ public class ReplicationPeerImpl implements ReplicationPeer
{
* @param id string representation of this peer's identifier
* @param peerConfig configuration for the replication peer
*/
- public ReplicationPeerImpl(Configuration conf, String id, boolean peerState,
- ReplicationPeerConfig peerConfig) {
+ public ReplicationPeerImpl(Configuration conf, String id,
ReplicationPeerConfig peerConfig,
+ boolean peerState, SyncReplicationState syncReplicationState) {
this.conf = conf;
this.id = id;
this.peerState = peerState ? PeerState.ENABLED : PeerState.DISABLED;
this.peerConfig = peerConfig;
+this.syncReplicationState = syncReplicationState;
this.peerConfigListeners = new ArrayList<>();
}
@@ -77,37 +80,26 @@ public class ReplicationPeerImpl implements ReplicationPeer
{
return peerState;
}
- /**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
+ @Override
+ public SyncReplicationState getSyncReplicationState() {
+return syncReplicationState;
+ }
+
@Override
public ReplicationPeerConfig getPeerConfig() {
return peerConfig;
}
- /**
- * Get the configuration object required to communicate with this peer
- * @return configuration object
- */
@Override
public Configuration getConfiguration() {
return conf;
}
- /**
- * Get replicable (table, cf-list) map of
[27/50] [abbrv] hbase git commit: HBASE-18133 Decrease quota reaction latency by HBase
HBASE-18133 Decrease quota reaction latency by HBase
Certain operations in HBase are known to directly affect
the utilization of tables on HDFS. When these actions
occur, we can circumvent the normal path and notify the
Master directly. This results in a much faster response to
changes in HDFS usage.
This requires FS scanning by the RS to be decoupled from
the reporting of sizes to the Master. An API inside each
RS is made so that any operation can hook into this call
in the face of other operations (e.g. compaction, flush,
bulk load).
Signed-off-by: Ted Yu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdedcc56
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdedcc56
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdedcc56
Branch: refs/heads/HBASE-19064
Commit: bdedcc5631fa9c8c400d4daa01b8a1947d4a12dd
Parents: 393ab30
Author: Josh Elser
Authored: Wed Feb 21 18:27:51 2018 -0500
Committer: Josh Elser
Committed: Wed Feb 28 14:11:20 2018 -0500
--
.../MetricsRegionServerQuotaSource.java | 24 ++
.../MetricsRegionServerQuotaSourceImpl.java | 86 +++
regionserver.MetricsRegionServerQuotaSource | 18 ++
.../quotas/FileSystemUtilizationChore.java | 39 +---
.../hbase/quotas/NoOpRegionSizeStore.java | 76 +++
.../quotas/RegionServerSpaceQuotaManager.java | 22 ++
.../apache/hadoop/hbase/quotas/RegionSize.java | 50
.../hadoop/hbase/quotas/RegionSizeImpl.java | 70 ++
.../hbase/quotas/RegionSizeReportingChore.java | 156 +
.../hadoop/hbase/quotas/RegionSizeStore.java| 82 +++
.../hbase/quotas/RegionSizeStoreFactory.java| 38
.../hbase/quotas/RegionSizeStoreImpl.java | 105 +
.../quotas/SpaceViolationPolicyEnforcement.java | 7 +-
.../AbstractViolationPolicyEnforcement.java | 32 ++-
.../DefaultViolationPolicyEnforcement.java | 34 +--
...ssingSnapshotViolationPolicyEnforcement.java | 8 +-
.../hadoop/hbase/regionserver/HRegion.java | 11 +
.../hbase/regionserver/HRegionServer.java | 36 ++-
.../hadoop/hbase/regionserver/HStore.java | 40
.../hbase/regionserver/MetricsRegionServer.java | 17 ++
.../hbase/regionserver/RSRpcServices.java | 19 +-
.../regionserver/RegionServerServices.java | 9 +
.../hadoop/hbase/MockRegionServerServices.java | 6 +
.../hadoop/hbase/master/MockRegionServer.java | 6 +
.../hbase/quotas/SpaceQuotaHelperForTests.java | 70 +-
.../quotas/TestFileSystemUtilizationChore.java | 27 +--
.../hbase/quotas/TestLowLatencySpaceQuotas.java | 228 +++
.../TestQuotaObserverChoreRegionReports.java| 17 +-
.../hadoop/hbase/quotas/TestRegionSizeImpl.java | 49
.../quotas/TestRegionSizeReportingChore.java| 127 +++
.../hbase/quotas/TestRegionSizeStoreImpl.java | 101
.../hadoop/hbase/quotas/TestSpaceQuotas.java| 50 +---
...kLoadCheckingViolationPolicyEnforcement.java | 8 +-
.../hadoop/hbase/regionserver/TestHStore.java | 51 +
.../TestRegionServerRegionSpaceUseReport.java | 53 ++---
35 files changed, 1575 insertions(+), 197 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdedcc56/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerQuotaSource.java
--
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerQuotaSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerQuotaSource.java
index 3175aad..5463a6a 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerQuotaSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerQuotaSource.java
@@ -28,6 +28,14 @@ public interface MetricsRegionServerQuotaSource extends
BaseSource {
String METRICS_DESCRIPTION = "Metrics about HBase RegionServer Quotas";
String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME;
+ String NUM_TABLES_IN_VIOLATION_NAME = "numTablesInViolation";
+ String NUM_SPACE_SNAPSHOTS_RECEIVED_NAME = "numSpaceSnapshotsReceived";
+ String FILE_SYSTEM_UTILIZATION_CHORE_TIME = "fileSystemUtilizationChoreTime";
+ String SPACE_QUOTA_REFRESHER_CHORE_TIME = "spaceQuotaRefresherChoreTime";
+
+ String NUM_REGION_SIZE_REPORT_NAME = "numRegionSizeReports";
+ String REGION_SIZE_REPORTING_CHORE_TIME_NAME =
"regionSizeReportingChoreTime";
+
/**
* Updates the metric tracking how many tables this RegionServer has marked
as in violation
* of their space quota.
@@ -57,4 +65,20 @@ public interface MetricsRegionServerQuotaSource extends
BaseSour
[28/50] [abbrv] hbase git commit: HBASE-20107 Add a test case for HBASE-14317 (Zephyr Guo)
HBASE-20107 Add a test case for HBASE-14317 (Zephyr Guo)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d7adc58e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d7adc58e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d7adc58e
Branch: refs/heads/HBASE-19064
Commit: d7adc58e5203567b8083160d45f85f9986e272cd
Parents: bdedcc5
Author: tedyu
Authored: Wed Feb 28 21:17:03 2018 -0800
Committer: tedyu
Committed: Wed Feb 28 21:17:03 2018 -0800
--
.../hadoop/hbase/regionserver/wal/FSHLog.java | 6 +-
.../hbase/regionserver/TestWALLockup.java | 135 +++
2 files changed, 139 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d7adc58e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 09bec14..6b77e80 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -679,7 +679,8 @@ public class FSHLog extends AbstractFSWAL {
return logRollNeeded;
}
- private long getSequenceOnRingBuffer() {
+ @VisibleForTesting
+ protected long getSequenceOnRingBuffer() {
return this.disruptor.getRingBuffer().next();
}
@@ -688,7 +689,8 @@ public class FSHLog extends AbstractFSWAL {
return publishSyncOnRingBuffer(sequence);
}
- private SyncFuture publishSyncOnRingBuffer(long sequence) {
+ @VisibleForTesting
+ protected SyncFuture publishSyncOnRingBuffer(long sequence) {
// here we use ring buffer sequence as transaction id
SyncFuture syncFuture = getSyncFuture(sequence);
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/d7adc58e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 5eb8fa8..9e051ae 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.wal.WALProvider.Writer;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
@@ -308,6 +309,140 @@ public class TestWALLockup {
}
}
+ /**
+ *
+ * If below is broken, we will see this test timeout because
RingBufferEventHandler was stuck in
+ * attainSafePoint. Everyone will wait for sync to finish forever. See
HBASE-14317.
+ */
+ @Test (timeout=3)
+ public void testRingBufferEventHandlerStuckWhenSyncFailed()
+throws IOException, InterruptedException {
+
+// A WAL that we can have throw exceptions and slow FSHLog.replaceWriter
down
+class DodgyFSLog extends FSHLog {
+
+ private volatile boolean zigZagCreated = false;
+
+ public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration
conf)
+throws IOException {
+super(fs, root, logDir, conf);
+ }
+
+ @Override
+ protected void afterCreatingZigZagLatch() {
+zigZagCreated = true;
+// Sleep a while to wait for RingBufferEventHandler to get stuck first.
+try {
+ Thread.sleep(3000);
+} catch (InterruptedException ignore) {
+}
+ }
+
+ @Override
+ protected long getSequenceOnRingBuffer() {
+return super.getSequenceOnRingBuffer();
+ }
+
+ protected void publishSyncOnRingBufferAndBlock(long sequence) {
+try {
+ super.blockOnSync(super.publishSyncOnRingBuffer(sequence));
+ Assert.fail("Expect an IOException here.");
+} catch (IOException ignore) {
+ // Here, we will get an IOException.
+}
+ }
+
+ @Override
+ protected Writer createWriterInstance(Path path) throws IOException {
+final Writer w = super.createWriterInstance(path);
+return new Writer() {
+ @Override
+ public void close() throws IOException {
+w.close();
+ }
+
+ @Override
+ public void sync() throws IOException {
+throw new IOException("FAKE! Failed to replace a bad
datanode...SYNC");
[12/50] [abbrv] hbase git commit: HBASE-20066 Region sequence id may go backward after split or merge
HBASE-20066 Region sequence id may go backward after split or merge
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f06a89b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f06a89b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f06a89b5
Branch: refs/heads/HBASE-19064
Commit: f06a89b5317a8e3d4fb220eb462b083bc6e0b5c8
Parents: d272ac9
Author: zhangduo
Authored: Mon Feb 26 20:22:57 2018 +0800
Committer: zhangduo
Committed: Tue Feb 27 15:33:07 2018 +0800
--
.../src/main/protobuf/MasterProcedure.proto | 22 ++-
.../assignment/MergeTableRegionsProcedure.java | 181 +--
.../assignment/SplitTableRegionProcedure.java | 150 +++
.../AbstractStateMachineTableProcedure.java | 14 +-
.../hadoop/hbase/regionserver/HRegion.java | 18 +-
.../apache/hadoop/hbase/wal/WALSplitter.java| 109 ++-
.../TestSequenceIdMonotonicallyIncreasing.java | 156
.../hadoop/hbase/master/AbstractTestDLS.java| 36
.../hbase/wal/TestReadWriteSeqIdFiles.java | 95 ++
9 files changed, 504 insertions(+), 277 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 1ab51e5..e785f96 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -241,11 +241,12 @@ enum SplitTableRegionState {
SPLIT_TABLE_REGION_PRE_OPERATION = 2;
SPLIT_TABLE_REGION_CLOSE_PARENT_REGION = 3;
SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS = 4;
- SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 5;
- SPLIT_TABLE_REGION_UPDATE_META = 6;
- SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 7;
- SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 8;
- SPLIT_TABLE_REGION_POST_OPERATION = 9;
+ SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE = 5;
+ SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META = 6;
+ SPLIT_TABLE_REGION_UPDATE_META = 7;
+ SPLIT_TABLE_REGION_PRE_OPERATION_AFTER_META = 8;
+ SPLIT_TABLE_REGION_OPEN_CHILD_REGIONS = 9;
+ SPLIT_TABLE_REGION_POST_OPERATION = 10;
}
message SplitTableRegionStateData {
@@ -260,11 +261,12 @@ enum MergeTableRegionsState {
MERGE_TABLE_REGIONS_PRE_MERGE_OPERATION = 3;
MERGE_TABLE_REGIONS_CLOSE_REGIONS = 4;
MERGE_TABLE_REGIONS_CREATE_MERGED_REGION = 5;
- MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 6;
- MERGE_TABLE_REGIONS_UPDATE_META = 7;
- MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 8;
- MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 9;
- MERGE_TABLE_REGIONS_POST_OPERATION = 10;
+ MERGE_TABLE_REGIONS_WRITE_MAX_SEQUENCE_ID_FILE = 6;
+ MERGE_TABLE_REGIONS_PRE_MERGE_COMMIT_OPERATION = 7;
+ MERGE_TABLE_REGIONS_UPDATE_META = 8;
+ MERGE_TABLE_REGIONS_POST_MERGE_COMMIT_OPERATION = 9;
+ MERGE_TABLE_REGIONS_OPEN_MERGED_REGION = 10;
+ MERGE_TABLE_REGIONS_POST_OPERATION = 11;
}
message MergeTableRegionsStateData {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f06a89b5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 1c448dc..7c041e7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -204,55 +205,55 @@ public class MergeTableRegionsProcedure
}
@Override
- protected Flow executeFromState(
- final MasterProcedureEnv env,
- final MergeTableRegionsState state)
- throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
-if (LOG.isDebugEnabled()) {
- LOG.debug(this + " execute state=" + state);
-}
+ protected Flow executeFromState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
+ throws ProcedureSuspendedException, ProcedureYieldException,
Interru
[30/50] [abbrv] hbase git commit: HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
HBASE-19598 Fix TestAssignmentManagerMetrics flaky test
Master never left waitForMasterActive because it never checked state of
the clusterUp flag. The test here was aborting regionserver and then
just exiting. The minihbasecluster shutdown sets the cluster down flag
but we were never looking at it so Master thread was staying up.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
The tableOnMaster check in waitForMasterActive looks wrong. It was
making it so a 'normal' Master was getting stuck in here. This is not
the place to worry about tablesOnMaster. That is for the balancer to be
concerned with. There is a problem with Master hosting
system-tables-only. After further study, Master can carry regions like a
regionserver but making it so it carries system tables only is tricky
given meta assign happens ahead of all others which means that the
Master needs to have checked-in as a regionserver super early... It
needs work. Punted for now.
M
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
Mostly renaming so lists and maps of region infos have same name as they
have elsewhere in code base and cleaning up confusion that may arise
when we talk of servers-for-system-tablesIt is talking about
something else in the code changes here that is other than the normal
understanding. It is about filtering regionservers by their version
numbers so we favor regions with higher version numbers. Needs to go
back up into the balancer.
M
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
It was possible for the Master to be given regions if no regionservers
available (as per the failing unit test in this case).
M
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Minor reordering moving the waitForMasterActive later in the initialize
and wrapping each test in a check if we are to keep looping (which
checks cluster status flag).
M
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
This was an old test from the days when Master carried system tables.
Updated test and fixed metrics. Metrics count the hbase:meta along with
the userspace region so upped expected numbers (previously the
hbase:meta was hosted on the master so metrics were not incremented).
M
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
I took a look at this test again but nope, needs a load of work still to
make it pass.
M hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
Stop being so whiney.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/016cf0c6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/016cf0c6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/016cf0c6
Branch: refs/heads/HBASE-19064
Commit: 016cf0c64ba867f740a19472f0fe716ee67306b1
Parents: 372c88a
Author: Michael Stack
Authored: Wed Feb 28 22:08:49 2018 -0800
Committer: Michael Stack
Committed: Wed Feb 28 22:25:24 2018 -0800
--
.../apache/hadoop/hbase/ipc/RpcExecutor.java| 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 10 ++--
.../hadoop/hbase/master/ServerManager.java | 2 +-
.../hadoop/hbase/master/SplitLogManager.java| 6 +--
.../master/assignment/AssignmentManager.java| 57 +++-
.../hbase/master/balancer/BaseLoadBalancer.java | 10 +++-
.../hbase/regionserver/HRegionServer.java | 34 ++--
.../hadoop/hbase/TestClientClusterMetrics.java | 8 ++-
.../hadoop/hbase/TestClientClusterStatus.java | 2 +-
.../master/TestAssignmentManagerMetrics.java| 39 ++
.../balancer/TestRegionsOnMasterOptions.java| 5 +-
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 10 ++--
12 files changed, 91 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/016cf0c6/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index 7470758..c3f613e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -327,7 +327,7 @@ public abstract class RpcExecutor {
int failedCount = failedHandlerCount.incrementAndGet();
if (this.handlerFailureThreshhold >= 0
&& failedCount > handlerCount * this.handlerFailureThreshhold) {
-String message = "Number of failed RpcServer handler runs exceeded
threshhold "
+String message = "N
[13/50] [abbrv] hbase git commit: HBASE-20092 Fix TestRegionMetrics#testRegionMetrics
HBASE-20092 Fix TestRegionMetrics#testRegionMetrics
Signed-off-by: Michael Stack
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dbd80130
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dbd80130
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dbd80130
Branch: refs/heads/HBASE-19064
Commit: dbd80130578ef2ed3ab1244090d36eab55ef32e6
Parents: f06a89b
Author: Chia-Ping Tsai
Authored: Tue Feb 27 16:48:17 2018 +0800
Committer: Chia-Ping Tsai
Committed: Tue Feb 27 16:48:17 2018 +0800
--
.../org/apache/hadoop/hbase/TestRegionLoad.java | 15 ++
.../apache/hadoop/hbase/TestRegionMetrics.java | 29 +---
2 files changed, 34 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbd80130/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java
index d0484d6..a390aca 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionLoad.java
@@ -27,6 +27,7 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
import org.apache.hadoop.hbase.client.Admin;
@@ -34,7 +35,6 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -61,12 +61,13 @@ public class TestRegionLoad {
private static final TableName TABLE_2 = TableName.valueOf("table_2");
private static final TableName TABLE_3 = TableName.valueOf("table_3");
private static final TableName[] tables = new TableName[]{TABLE_1, TABLE_2,
TABLE_3};
+ private static final int MSG_INTERVAL = 500; // ms
@BeforeClass
public static void beforeClass() throws Exception {
// Make servers report eagerly. This test is about looking at the cluster
status reported.
// Make it so we don't have to wait around too long to see change.
-UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 500);
+UTIL.getConfiguration().setInt("hbase.regionserver.msginterval",
MSG_INTERVAL);
UTIL.startMiniCluster(4);
admin = UTIL.getAdmin();
admin.setBalancerRunning(false, true);
@@ -117,11 +118,13 @@ public class TestRegionLoad {
}
checkRegionsAndRegionLoads(tableRegions, regionLoads);
}
-int pause =
UTIL.getConfiguration().getInt("hbase.regionserver.msginterval", 3000);
// Just wait here. If this fixes the test, come back and do a better job.
// Would have to redo the below so can wait on cluster status changing.
-Threads.sleep(2 * pause);
+// Admin#getClusterMetrics retrieves data from HMaster.
Admin#getRegionMetrics, by contrast,
+// get the data from RS. Hence, it will fail if we do the assert check
before RS has done
+// the report.
+TimeUnit.MILLISECONDS.sleep(3 * MSG_INTERVAL);
// Check RegionLoad matches the regionLoad from ClusterStatus
ClusterStatus clusterStatus
@@ -133,10 +136,10 @@ public class TestRegionLoad {
(v1, v2) -> {
throw new RuntimeException("impossible!!");
}, () -> new TreeMap<>(Bytes.BYTES_COMPARATOR)));
- LOG.info("serverName=" + serverName + ", getRegionLoads=" +
+ LOG.debug("serverName=" + serverName + ", getRegionLoads=" +
serverLoad.getRegionsLoad().keySet().stream().map(r ->
Bytes.toString(r)).
collect(Collectors.toList()));
- LOG.info("serverName=" + serverName + ", regionLoads=" +
+ LOG.debug("serverName=" + serverName + ", regionLoads=" +
regionLoads.keySet().stream().map(r -> Bytes.toString(r)).
collect(Collectors.toList()));
compareRegionLoads(serverLoad.getRegionsLoad(), regionLoads);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbd80130/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionMetrics.java
index 1f9c519..df57c49 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/
[26/50] [abbrv] hbase git commit: HBASE-18133 Decrease quota reaction latency by HBase
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdedcc56/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
new file mode 100644
index 000..f90ed82
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ClientServiceCallable;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RpcRetryingCaller;
+import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
+import
org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+
+@Category({MediumTests.class})
+public class TestLowLatencySpaceQuotas {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestLowLatencySpaceQuotas.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ // Global for all tests in the class
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+ private Connection conn;
+ private Admin admin;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+// The default 1s period for QuotaObserverChore is good.
+SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+// Set the period to read region size from HDFS to be very long
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000 * 120);
+TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
+conn = TEST_UTIL.getConnection();
+admin = TEST_UTIL.getAdmin();
+helper.waitForQuotaTable(conn);
+ }
+
+ @Test
+ public void testFlushes() throws Exception {
+TableName tn = helper.createTableWithRegions(1);
+// Set a quota
+QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
+tn, SpaceQuotaHelperForTests.ONE_GIGABYTE,
SpaceViolationPolicy.NO_INSERTS);
+admin.setQuota(settings);
+
+// Write some data
+final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+helper.writeData(tn, initialSize);
+
+// Make sure a flush happened
+admin.flush(tn);
+
+// We should be able to observe the system recording an increase in size
(even
+// though we know the filesystem scanning did not happen).
+TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn,
tn) {
+ @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception
{
+return snapshot.getUsage() >= initialSize;
+ }
+});
+ }
+
[48/50] [abbrv] hbase git commit: HBASE-19864 Use protobuf instead of enum.ordinal to store SyncReplicationState
HBASE-19864 Use protobuf instead of enum.ordinal to store SyncReplicationState
Signed-off-by: zhangduo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c6a2c9da
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c6a2c9da
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c6a2c9da
Branch: refs/heads/HBASE-19064
Commit: c6a2c9daa09ae55ccc7faafea7b1de6183f1333d
Parents: f1ee86b
Author: Guanghao Zhang
Authored: Fri Jan 26 16:50:48 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../replication/ReplicationPeerConfigUtil.java | 22 ++---
.../hbase/replication/SyncReplicationState.java | 17 +
.../hbase/shaded/protobuf/RequestConverter.java | 7 +++---
.../src/main/protobuf/Replication.proto | 13 ++
.../replication/ZKReplicationPeerStorage.java | 25 +---
.../hadoop/hbase/master/MasterRpcServices.java | 9 ---
...ransitPeerSyncReplicationStateProcedure.java | 9 ---
.../TestReplicationSourceManager.java | 2 +-
8 files changed, 67 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c6a2c9da/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 86b49ea..5096824 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -398,7 +398,7 @@ public final class ReplicationPeerConfigUtil {
ReplicationProtos.ReplicationState.State.ENABLED ==
desc.getState().getState();
ReplicationPeerConfig config = convert(desc.getConfig());
return new ReplicationPeerDescription(desc.getId(), enabled, config,
-
SyncReplicationState.valueOf(desc.getSyncReplicationState().getNumber()));
+ toSyncReplicationState(desc.getSyncReplicationState()));
}
public static ReplicationProtos.ReplicationPeerDescription
@@ -406,17 +406,33 @@ public final class ReplicationPeerConfigUtil {
ReplicationProtos.ReplicationPeerDescription.Builder builder =
ReplicationProtos.ReplicationPeerDescription.newBuilder();
builder.setId(desc.getPeerId());
+
ReplicationProtos.ReplicationState.Builder stateBuilder =
ReplicationProtos.ReplicationState.newBuilder();
stateBuilder.setState(desc.isEnabled() ?
ReplicationProtos.ReplicationState.State.ENABLED :
ReplicationProtos.ReplicationState.State.DISABLED);
builder.setState(stateBuilder.build());
+
builder.setConfig(convert(desc.getPeerConfig()));
-builder.setSyncReplicationState(
-
ReplicationProtos.SyncReplicationState.forNumber(desc.getSyncReplicationState().ordinal()));
+
builder.setSyncReplicationState(toSyncReplicationState(desc.getSyncReplicationState()));
+
return builder.build();
}
+ public static ReplicationProtos.SyncReplicationState
+ toSyncReplicationState(SyncReplicationState state) {
+ReplicationProtos.SyncReplicationState.Builder syncReplicationStateBuilder
=
+ReplicationProtos.SyncReplicationState.newBuilder();
+syncReplicationStateBuilder
+
.setState(ReplicationProtos.SyncReplicationState.State.forNumber(state.ordinal()));
+return syncReplicationStateBuilder.build();
+ }
+
+ public static SyncReplicationState
+ toSyncReplicationState(ReplicationProtos.SyncReplicationState state) {
+return SyncReplicationState.valueOf(state.getState().getNumber());
+ }
+
public static ReplicationPeerConfig appendTableCFsToReplicationPeerConfig(
Map> tableCfs, ReplicationPeerConfig peerConfig)
{
ReplicationPeerConfigBuilder builder =
ReplicationPeerConfig.newBuilder(peerConfig);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c6a2c9da/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
index bd144e9..a65b144 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/SyncReplicationState.java
@@ -17,8 +17,15 @@
*/
package org.apache.hadoop.hbase.replication;
+impo
[23/50] [abbrv] hbase git commit: HBASE-17825: Backup further optimizations
HBASE-17825: Backup further optimizations
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6cfa208a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6cfa208a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6cfa208a
Branch: refs/heads/HBASE-19064
Commit: 6cfa208add6ea424e17cae00114ebd3e7d7967f1
Parents: 62ee7d9
Author: Vladimir Rodionov
Authored: Tue Feb 27 14:08:14 2018 -0800
Committer: Josh Elser
Committed: Wed Feb 28 09:14:15 2018 -0500
--
.../impl/IncrementalTableBackupClient.java | 14 ++-
.../mapreduce/MapReduceBackupMergeJob.java | 60 --
.../TestIncrementalBackupMergeWithFailures.java | 4 +-
.../hbase/mapreduce/HFileOutputFormat2.java | 18 ++-
.../hadoop/hbase/mapreduce/WALPlayer.java | 114 ++-
.../hadoop/hbase/util/FSTableDescriptors.java | 6 +-
6 files changed, 142 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6cfa208a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
--
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
index c897ae2..8fd6573 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
@@ -374,14 +374,17 @@ public class IncrementalTableBackupClient extends
TableBackupClient {
Set tableSet = backupManager.getIncrementalBackupTableSet();
// filter missing files out (they have been copied by previous backups)
incrBackupFileList = filterMissingFiles(incrBackupFileList);
+List tableList = new ArrayList();
for (TableName table : tableSet) {
// Check if table exists
if (tableExists(table, conn)) {
-walToHFiles(incrBackupFileList, table);
+tableList.add(table.getNameAsString());
} else {
LOG.warn("Table " + table + " does not exists. Skipping in WAL
converter");
}
}
+walToHFiles(incrBackupFileList, tableList);
+
}
protected boolean tableExists(TableName table, Connection conn) throws
IOException {
@@ -390,20 +393,21 @@ public class IncrementalTableBackupClient extends
TableBackupClient {
}
}
- protected void walToHFiles(List dirPaths, TableName tableName)
throws IOException {
+ protected void walToHFiles(List dirPaths, List tableList)
throws IOException {
Tool player = new WALPlayer();
// Player reads all files in arbitrary directory structure and creates
// a Map task for each file. We use ';' as separator
// because WAL file names contains ','
String dirs = StringUtils.join(dirPaths, ';');
-String jobname = "Incremental_Backup-" + backupId + "-" +
tableName.getNameAsString();
+String jobname = "Incremental_Backup-" + backupId ;
-Path bulkOutputPath = getBulkOutputDirForTable(tableName);
+Path bulkOutputPath = getBulkOutputDir();
conf.set(WALPlayer.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString());
conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";");
+conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true);
conf.set(JOB_NAME_CONF_KEY, jobname);
-String[] playerArgs = { dirs, tableName.getNameAsString() };
+String[] playerArgs = { dirs, StringUtils.join(tableList, ",") };
try {
player.setConf(conf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6cfa208a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java
--
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java
index 6f2c44c..3fcf692 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.backup.mapreduce;
import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
@@ -29,7 +30,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.
[39/50] [abbrv] hbase git commit: HBASE-19083 Introduce a new log writer which can write to two HDFSes
HBASE-19083 Introduce a new log writer which can write to two HDFSes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc7fee0f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc7fee0f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc7fee0f
Branch: refs/heads/HBASE-19064
Commit: fc7fee0fc088dc909d76eec8d94667e22baa674a
Parents: c415ddb
Author: zhangduo
Authored: Thu Jan 11 21:08:02 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:25:13 2018 +0800
--
.../hbase/regionserver/wal/AsyncFSWAL.java | 21 +--
.../regionserver/wal/CombinedAsyncWriter.java | 134 ++
.../hbase/regionserver/wal/DualAsyncFSWAL.java | 67 +
.../wal/AbstractTestProtobufLog.java| 110 +++
.../regionserver/wal/ProtobufLogTestHelper.java | 99 ++
.../regionserver/wal/TestAsyncProtobufLog.java | 32 +
.../wal/TestCombinedAsyncWriter.java| 136 +++
.../hbase/regionserver/wal/TestProtobufLog.java | 14 +-
.../regionserver/wal/WriterOverAsyncWriter.java | 63 +
9 files changed, 533 insertions(+), 143 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fc7fee0f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index e34818f..0bee9d6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -607,12 +607,16 @@ public class AsyncFSWAL extends
AbstractFSWAL {
}
}
- @Override
- protected AsyncWriter createWriterInstance(Path path) throws IOException {
+ protected final AsyncWriter createAsyncWriter(FileSystem fs, Path path)
throws IOException {
return AsyncFSWALProvider.createAsyncWriter(conf, fs, path, false,
eventLoopGroup,
channelClass);
}
+ @Override
+ protected AsyncWriter createWriterInstance(Path path) throws IOException {
+return createAsyncWriter(fs, path);
+ }
+
private void waitForSafePoint() {
consumeLock.lock();
try {
@@ -632,13 +636,12 @@ public class AsyncFSWAL extends
AbstractFSWAL {
}
}
- private long closeWriter() {
-AsyncWriter oldWriter = this.writer;
-if (oldWriter != null) {
- long fileLength = oldWriter.getLength();
+ protected final long closeWriter(AsyncWriter writer) {
+if (writer != null) {
+ long fileLength = writer.getLength();
closeExecutor.execute(() -> {
try {
- oldWriter.close();
+ writer.close();
} catch (IOException e) {
LOG.warn("close old writer failed", e);
}
@@ -654,7 +657,7 @@ public class AsyncFSWAL extends AbstractFSWAL {
throws IOException {
Preconditions.checkNotNull(nextWriter);
waitForSafePoint();
-long oldFileLen = closeWriter();
+long oldFileLen = closeWriter(this.writer);
logRollAndSetupWalProps(oldPath, newPath, oldFileLen);
this.writer = nextWriter;
if (nextWriter instanceof AsyncProtobufLogWriter) {
@@ -679,7 +682,7 @@ public class AsyncFSWAL extends AbstractFSWAL {
@Override
protected void doShutdown() throws IOException {
waitForSafePoint();
-closeWriter();
+closeWriter(this.writer);
closeExecutor.shutdown();
try {
if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds,
TimeUnit.SECONDS)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/fc7fee0f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
new file mode 100644
index 000..8ecfede
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by appl
[42/50] [abbrv] hbase git commit: HBASE-19781 Add a new cluster state flag for synchronous replication
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7d22055/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
index 8911982..f5eca39 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
@@ -28,6 +28,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
@@ -67,9 +68,9 @@ public class TestHBaseFsckReplication {
String peerId1 = "1";
String peerId2 = "2";
peerStorage.addPeer(peerId1,
ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true);
+ true, SyncReplicationState.NONE);
peerStorage.addPeer(peerId2,
ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true);
+ true, SyncReplicationState.NONE);
for (int i = 0; i < 10; i++) {
queueStorage.addWAL(ServerName.valueOf("localhost", 1 + i, 10 +
i), peerId1,
"file-" + i);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7d22055/hbase-shell/src/main/ruby/hbase/replication_admin.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/replication_admin.rb
b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
index ba7d191..d5d4844 100644
--- a/hbase-shell/src/main/ruby/hbase/replication_admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
@@ -20,6 +20,7 @@
include Java
java_import
org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil
+java_import org.apache.hadoop.hbase.replication.SyncReplicationState
java_import org.apache.hadoop.hbase.replication.ReplicationPeerConfig
java_import org.apache.hadoop.hbase.util.Bytes
java_import org.apache.hadoop.hbase.zookeeper.ZKConfig
@@ -329,6 +330,20 @@ module Hbase
'!' + ReplicationPeerConfigUtil.convertToString(tableCFs)
end
+# Transit current cluster to a new state in the specified synchronous
+# replication peer
+def transit_peer_sync_replication_state(id, state)
+ if 'ACTIVE'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id,
SyncReplicationState::ACTIVE)
+ elsif 'DOWNGRADE_ACTIVE'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id,
SyncReplicationState::DOWNGRADE_ACTIVE)
+ elsif 'STANDBY'.eql?(state)
+@admin.transitReplicationPeerSyncReplicationState(id,
SyncReplicationState::STANDBY)
+ else
+raise(ArgumentError, 'synchronous replication state must be ACTIVE,
DOWNGRADE_ACTIVE or STANDBY')
+ end
+end
+
#--
# Enables a table's replication switch
def enable_tablerep(table_name)
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7d22055/hbase-shell/src/main/ruby/shell.rb
--
diff --git a/hbase-shell/src/main/ruby/shell.rb
b/hbase-shell/src/main/ruby/shell.rb
index 507c0a9..0ed71ae 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -395,6 +395,7 @@ Shell.load_command_group(
get_peer_config
list_peer_configs
update_peer_config
+transit_peer_sync_replication_state
]
)
http://git-wip-us.apache.org/repos/asf/hbase/blob/c7d22055/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
index caeab86..aa10fda 100644
--- a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
@@ -39,8 +39,8 @@ EOF
peers = replication_admin.list_peers
formatter.header(%w[PEER_ID CLUSTER_KEY ENDPOINT_CLASSNAME
-REMOTE_ROOT_DIR STATE REPLICATE_ALL
-NAMESPACES TABLE_CFS BANDWIDTH])
+REMOTE_ROOT_DIR SYNC_REPLICATION_STATE STATE
+REPLICATE_ALL NAMESPACES TABLE_CFS BANDWIDTH])
peers.each
[08/50] [abbrv] hbase git commit: for creating patch HBASE-20074-V01.patch
for creating patch HBASE-20074-V01.patch
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73028d5b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73028d5b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73028d5b
Branch: refs/heads/HBASE-19064
Commit: 73028d5bd9f85655b284654579ddcbbca31e41e8
Parents: 7cfb464
Author: gsheffi
Authored: Mon Feb 26 11:18:38 2018 +0200
Committer: Michael Stack
Committed: Mon Feb 26 09:57:22 2018 -0800
--
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/73028d5b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 44b40eb..d60b049 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
-// multiply by a factor (different factors for different index types)
-if (indexType == IndexType.ARRAY_MAP) {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+// multiply by a factor (the same factor for all index types)
+factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-} else {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
- IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-}
+
inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to {} and immutable
segments index to type={}",
StringUtils.byteDesc(inmemoryFlushSize), indexType);
[44/50] [abbrv] hbase git commit: HBASE-19957 General framework to transit sync replication state
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b2faaca/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index ebb8042..b73733c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -50,6 +50,7 @@ import
org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationListener;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
@@ -131,6 +132,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
// For recovered source, the queue id's format is peer_id-servername-*
private final ConcurrentMap>>
walsByIdRecoveredQueues;
+ private final SyncReplicationPeerMappingManager
syncReplicationPeerMappingManager;
+
private final Configuration conf;
private final FileSystem fs;
// The paths to the latest log of each wal group, for new coming peers
@@ -165,9 +168,8 @@ public class ReplicationSourceManager implements
ReplicationListener {
public ReplicationSourceManager(ReplicationQueueStorage queueStorage,
ReplicationPeers replicationPeers, ReplicationTracker
replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID
clusterId,
- WALFileLengthProvider walFileLengthProvider) throws IOException {
-// CopyOnWriteArrayList is thread-safe.
-// Generally, reading is more than modifying.
+ WALFileLengthProvider walFileLengthProvider,
+ SyncReplicationPeerMappingManager syncReplicationPeerMappingManager)
throws IOException {
this.sources = new ConcurrentHashMap<>();
this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
@@ -180,10 +182,11 @@ public class ReplicationSourceManager implements
ReplicationListener {
this.fs = fs;
this.logDir = logDir;
this.oldLogDir = oldLogDir;
-this.sleepBeforeFailover =
conf.getLong("replication.sleep.before.failover", 3); // 30
-
// seconds
+// 30 seconds
+this.sleepBeforeFailover =
conf.getLong("replication.sleep.before.failover", 3);
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
+this.syncReplicationPeerMappingManager = syncReplicationPeerMappingManager;
this.replicationTracker.registerListener(this);
// It's preferable to failover 1 RS at a time, but with good zk servers
// more could be processed at the same time.
@@ -244,8 +247,11 @@ public class ReplicationSourceManager implements
ReplicationListener {
}
/**
- * 1. Add peer to replicationPeers 2. Add the normal source and related
replication queue 3. Add
- * HFile Refs
+ *
+ * Add peer to replicationPeers
+ * Add the normal source and related replication queue
+ * Add HFile Refs
+ *
* @param peerId the id of replication peer
*/
public void addPeer(String peerId) throws IOException {
@@ -264,13 +270,16 @@ public class ReplicationSourceManager implements
ReplicationListener {
}
/**
- * 1. Remove peer for replicationPeers 2. Remove all the recovered sources
for the specified id
- * and related replication queues 3. Remove the normal source and related
replication queue 4.
- * Remove HFile Refs
+ *
+ * Remove peer for replicationPeers
+ * Remove all the recovered sources for the specified id and related
replication queues
+ * Remove the normal source and related replication queue
+ * Remove HFile Refs
+ *
* @param peerId the id of the replication peer
*/
public void removePeer(String peerId) {
-replicationPeers.removePeer(peerId);
+ReplicationPeer peer = replicationPeers.removePeer(peerId);
String terminateMessage = "Replication stream was removed by a user";
List oldSourcesToDelete = new ArrayList<>();
// synchronized on oldsources to avoid adding recovered source for the
to-be-removed peer
@@ -301,7 +310,10 @@ public class ReplicationSourceManager implements
ReplicationListener {
deleteQueue(peerId);
this.walsById.remove(peerId);
}
-
+Replicatio
[17/50] [abbrv] hbase git commit: HBASE-20074 [FindBugs] Same code on both branches in CompactingMemStore#initMemStoreCompactor
HBASE-20074 [FindBugs] Same code on both branches in
CompactingMemStore#initMemStoreCompactor
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0fa5d69f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0fa5d69f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0fa5d69f
Branch: refs/heads/HBASE-19064
Commit: 0fa5d69fc7602f2760a64e1760fd02be130aa8bd
Parents: e4ce38d
Author: gsheffi
Authored: Mon Feb 26 11:18:38 2018 +0200
Committer: Sean Busbey
Committed: Tue Feb 27 11:21:40 2018 -0600
--
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0fa5d69f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 44b40eb..d60b049 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -136,14 +136,10 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
-// multiply by a factor (different factors for different index types)
-if (indexType == IndexType.ARRAY_MAP) {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+// multiply by a factor (the same factor for all index types)
+factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-} else {
- factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
- IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
-}
+
inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to {} and immutable
segments index to type={}",
StringUtils.byteDesc(inmemoryFlushSize), indexType);
[45/50] [abbrv] hbase git commit: HBASE-19957 General framework to transit sync replication state
HBASE-19957 General framework to transit sync replication state Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b2faaca Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b2faaca Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b2faaca Branch: refs/heads/HBASE-19064 Commit: 3b2faaca43f9e09f3f26a0fc49f132eeb7a34615 Parents: c825551 Author: zhangduo Authored: Fri Feb 9 18:33:28 2018 +0800 Committer: zhangduo Committed: Fri Mar 2 14:28:32 2018 +0800 -- .../replication/ReplicationPeerConfig.java | 2 - .../replication/ReplicationPeerDescription.java | 5 +- .../hbase/replication/SyncReplicationState.java | 19 +- .../org/apache/hadoop/hbase/HConstants.java | 3 + .../src/main/protobuf/MasterProcedure.proto | 20 ++- .../hbase/replication/ReplicationPeerImpl.java | 45 - .../replication/ReplicationPeerStorage.java | 25 ++- .../hbase/replication/ReplicationPeers.java | 27 ++- .../replication/ZKReplicationPeerStorage.java | 65 +-- .../hbase/coprocessor/MasterObserver.java | 7 +- .../org/apache/hadoop/hbase/master/HMaster.java | 4 +- .../hbase/master/MasterCoprocessorHost.java | 12 +- .../replication/AbstractPeerProcedure.java | 14 +- .../master/replication/ModifyPeerProcedure.java | 15 +- .../replication/RefreshPeerProcedure.java | 18 +- .../replication/ReplicationPeerManager.java | 107 +++- ...ransitPeerSyncReplicationStateProcedure.java | 175 --- .../hbase/regionserver/HRegionServer.java | 35 ++-- .../regionserver/ReplicationSourceService.java | 11 +- .../regionserver/PeerActionListener.java| 4 +- .../regionserver/PeerProcedureHandler.java | 16 +- .../regionserver/PeerProcedureHandlerImpl.java | 55 +- .../regionserver/RefreshPeerCallable.java | 7 + .../replication/regionserver/Replication.java | 22 ++- .../regionserver/ReplicationSourceManager.java | 41 +++-- .../SyncReplicationPeerInfoProvider.java| 43 + .../SyncReplicationPeerInfoProviderImpl.java| 71 .../SyncReplicationPeerMappingManager.java | 48 + .../SyncReplicationPeerProvider.java| 35 .../hbase/wal/SyncReplicationWALProvider.java | 35 ++-- .../org/apache/hadoop/hbase/wal/WALFactory.java | 47 ++--- .../replication/TestReplicationAdmin.java | 3 +- .../TestReplicationSourceManager.java | 5 +- .../wal/TestSyncReplicationWALProvider.java | 36 ++-- 34 files changed, 752 insertions(+), 325 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/3b2faaca/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index 69565a7..79b3a1d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.replication; import java.util.Collection; @@ -25,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; http://git-wip-us.apache.org/repos/asf/hbase/blob/3b2faaca/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java index 2d077c5..b0c27bb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerDescription.java @@ -20,7 +20,10 @@ package org.apache.hadoop.hbase.replication; import org.apache.yetus.audience.InterfaceAudience; /** - * The POJO equivalent of ReplicationProtos.ReplicationPeerDescription + * The POJO equivalent of ReplicationProtos.ReplicationPeerDescription. + * + * To developer, here we do not store the new sync replication state since it is just an + * intermediate state and this class is public. */ @InterfaceAudience.Public pub
[31/50] [abbrv] hbase git commit: HBASE-20110 Findbugs in zk and mr caused nightly #409 branch-2 to fail
HBASE-20110 Findbugs in zk and mr caused nightly #409 branch-2 to fail
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5b64de45
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5b64de45
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5b64de45
Branch: refs/heads/HBASE-19064
Commit: 5b64de45edf85abdc32ed98cf4396dea43283377
Parents: 016cf0c
Author: Michael Stack
Authored: Wed Feb 28 23:38:35 2018 -0800
Committer: Michael Stack
Committed: Wed Feb 28 23:39:21 2018 -0800
--
.../src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java | 2 ++
.../org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java | 3 ---
2 files changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5b64de45/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
--
diff --git
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index d58ff48..f7405fd 100644
---
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -145,6 +145,8 @@ public class Import extends Configured implements Tool {
}
@Override
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"EQ_COMPARETO_USE_OBJECT_EQUALS",
+justification = "This is wrong, yes, but we should be purging
Writables, not fixing them")
public int compareTo(CellWritableComparable o) {
return CellComparator.getInstance().compare(this.kv, o.kv);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5b64de45/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
--
diff --git
a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index 0285a0d..9343ec2 100644
---
a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++
b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -419,7 +419,6 @@ public class RecoverableZooKeeper {
// try to verify whether the previous setData success or not
try{
Stat stat = new Stat();
- startTime = EnvironmentEdgeManager.currentTime();
byte[] revData = checkZk().getData(path, false, stat);
if(Bytes.compareTo(revData, newData) == 0) {
// the bad version is caused by previous successful setData
@@ -556,7 +555,6 @@ public class RecoverableZooKeeper {
// If the connection was lost, there is still a possibility that
// we have successfully created the node at our previous attempt,
// so we read the node and compare.
- startTime = EnvironmentEdgeManager.currentTime();
byte[] currentData = checkZk().getData(path, false, null);
if (currentData != null &&
Bytes.compareTo(currentData, data) == 0) {
@@ -725,7 +723,6 @@ public class RecoverableZooKeeper {
}
public void sync(String path, AsyncCallback.VoidCallback cb, Object ctx)
throws KeeperException {
-long startTime = EnvironmentEdgeManager.currentTime();
checkZk().sync(path, cb, null);
}
[04/50] [abbrv] hbase git commit: HBASE-20082 Fix findbugs errors only on master which are introduced by HBASE-19397
HBASE-20082 Fix findbugs errors only on master which are introduced by
HBASE-19397
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c74d177
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c74d177
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c74d177
Branch: refs/heads/HBASE-19064
Commit: 8c74d177f68bbd5412cef96dc33f16ba33ff7875
Parents: a34f129
Author: zhangduo
Authored: Mon Feb 26 10:22:09 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 15:50:51 2018 +0800
--
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 2 ++
.../hbase/replication/regionserver/ReplicationSourceManager.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 1253ef9..ba9bcdc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -50,6 +50,8 @@ public class RefreshPeerProcedure extends
Procedure
private PeerOperationType type;
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"IS2_INCONSISTENT_SYNC",
+ justification = "Will never change after construction")
private ServerName targetServer;
private boolean dispatched;
http://git-wip-us.apache.org/repos/asf/hbase/blob/8c74d177/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 4e1b20d..ebb8042 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -403,7 +403,7 @@ public class ReplicationSourceManager implements
ReplicationListener {
toStartup.add(replicationSource);
}
}
-for (ReplicationSourceInterface replicationSource : oldsources) {
+for (ReplicationSourceInterface replicationSource : toStartup) {
replicationSource.startup();
}
}
[02/50] [abbrv] hbase git commit: HBASE-20065 Addendum remove wrong comment
HBASE-20065 Addendum remove wrong comment
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a8471bd9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a8471bd9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a8471bd9
Branch: refs/heads/HBASE-19064
Commit: a8471bd98736c7ee387e268415bfd3ff96d8655d
Parents: 549a6d9
Author: zhangduo
Authored: Mon Feb 26 09:48:41 2018 +0800
Committer: zhangduo
Committed: Mon Feb 26 09:48:41 2018 +0800
--
.../src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java| 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8471bd9/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 803f183..e779054 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -1535,7 +1535,6 @@ public class MetaTableAccessor {
RegionInfo regionA, RegionInfo regionB, ServerName sn, int
regionReplication)
throws IOException {
try (Table meta = getMetaHTable(connection)) {
- // use the maximum of what master passed us vs local time.
long time = EnvironmentEdgeManager.currentTime();
// Put for parent
[32/50] [abbrv] hbase git commit: Revert "HBASE-19598 Fix TestAssignmentManagerMetrics flaky test" Pushed prematurely. Revert.
Revert "HBASE-19598 Fix TestAssignmentManagerMetrics flaky test"
Pushed prematurely. Revert.
This reverts commit 016cf0c64ba867f740a19472f0fe716ee67306b1.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0732ef5e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0732ef5e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0732ef5e
Branch: refs/heads/HBASE-19064
Commit: 0732ef5ebfea8e254d4a5dad6fe16ea3b799
Parents: 5b64de4
Author: Michael Stack
Authored: Wed Feb 28 23:39:54 2018 -0800
Committer: Michael Stack
Committed: Wed Feb 28 23:39:54 2018 -0800
--
.../apache/hadoop/hbase/ipc/RpcExecutor.java| 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 10 ++--
.../hadoop/hbase/master/ServerManager.java | 2 +-
.../hadoop/hbase/master/SplitLogManager.java| 6 ++-
.../master/assignment/AssignmentManager.java| 57 +---
.../hbase/master/balancer/BaseLoadBalancer.java | 10 +---
.../hbase/regionserver/HRegionServer.java | 34 ++--
.../hadoop/hbase/TestClientClusterMetrics.java | 8 +--
.../hadoop/hbase/TestClientClusterStatus.java | 2 +-
.../master/TestAssignmentManagerMetrics.java| 39 --
.../balancer/TestRegionsOnMasterOptions.java| 5 +-
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 10 ++--
12 files changed, 94 insertions(+), 91 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0732ef5e/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
index c3f613e..7470758 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcExecutor.java
@@ -327,7 +327,7 @@ public abstract class RpcExecutor {
int failedCount = failedHandlerCount.incrementAndGet();
if (this.handlerFailureThreshhold >= 0
&& failedCount > handlerCount * this.handlerFailureThreshhold) {
-String message = "Number of failed RpcServer handler runs exceeded
threshold "
+String message = "Number of failed RpcServer handler runs exceeded
threshhold "
+ this.handlerFailureThreshhold + "; reason: " +
StringUtils.stringifyException(e);
if (abortable != null) {
abortable.abort(message, e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0732ef5e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3d69888..b0dd0b4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -653,12 +653,14 @@ public class HMaster extends HRegionServer implements
MasterServices {
}
/**
- * Wait here if backup Master. This avoids showing backup masters as
regionservers in master
- * web UI, or assigning any region to them.
+ * If configured to put regions on active master,
+ * wait till a backup master becomes active.
+ * Otherwise, loop till the server is stopped or aborted.
*/
@Override
- protected void waitForMasterActive() {
-while (!this.activeMaster && keepLooping()) {
+ protected void waitForMasterActive(){
+boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf);
+while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) {
sleeper.sleep();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0732ef5e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 6488559..06d6c8b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -884,7 +884,7 @@ public class ServerManager {
if (isClusterShutdown()) {
this.master.stop("Cluster shutdown");
}
-LOG.info("RegionServer count=" + count + "; waited=" + slept + "ms," +
+LOG.info("Finished waiting on RegionServer count=" + count + "; waited=" +
slept + "ms," +
" expected min=" + minToStart + " server(s), max=" +
getStr
[24/50] [abbrv] hbase git commit: HBASE-20106 [api compliance chacker] Fix Bug Where Branch Isn't Found
HBASE-20106 [api compliance chacker] Fix Bug Where Branch Isn't Found While git rev-parse, sometimes the branch cannot be found unless the remote is specified. This fix tries to use "origin" if the remote is not specified and the branch is not found. Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96ebab74 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96ebab74 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96ebab74 Branch: refs/heads/HBASE-19064 Commit: 96ebab748fae24582cfaea0d0f8c318f79bf1681 Parents: 6cfa208 Author: Alex Leblang Authored: Thu Sep 7 12:31:53 2017 -0400 Committer: Sean Busbey Committed: Wed Feb 28 11:52:15 2018 -0600 -- dev-support/checkcompatibility.py | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/96ebab74/dev-support/checkcompatibility.py -- diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py index 83189cf..ea9c229 100755 --- a/dev-support/checkcompatibility.py +++ b/dev-support/checkcompatibility.py @@ -116,7 +116,12 @@ def checkout_java_tree(rev, path): def get_git_hash(revname): """ Convert 'revname' to its SHA-1 hash. """ -return check_output(["git", "rev-parse", revname], +try: +return check_output(["git", "rev-parse", revname], +cwd=get_repo_dir()).strip() +except: +revname = "origin/" + revname +return check_output(["git", "rev-parse", revname], cwd=get_repo_dir()).strip()
[50/50] [abbrv] hbase git commit: HBASE-19082 Reject read/write from client but accept write from replication in state S
HBASE-19082 Reject read/write from client but accept write from replication in
state S
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/010cc7d7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/010cc7d7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/010cc7d7
Branch: refs/heads/HBASE-19064
Commit: 010cc7d79966b48e848cb31cd0b5191c35003df7
Parents: 3b2faac
Author: zhangduo
Authored: Mon Feb 12 18:20:18 2018 +0800
Committer: zhangduo
Committed: Fri Mar 2 14:28:32 2018 +0800
--
.../org/apache/hadoop/hbase/HConstants.java | 3 -
.../src/main/protobuf/MasterProcedure.proto | 3 +-
.../hbase/replication/ReplicationUtils.java | 4 +
...ransitPeerSyncReplicationStateProcedure.java | 10 +
.../hadoop/hbase/regionserver/HRegion.java | 5 +-
.../hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/regionserver/RSRpcServices.java | 88 ++--
.../RejectRequestsFromClientStateChecker.java | 44
.../regionserver/ReplicationSink.java | 72 ---
.../SyncReplicationPeerInfoProvider.java| 10 +-
.../SyncReplicationPeerInfoProviderImpl.java| 19 +-
.../hbase/wal/SyncReplicationWALProvider.java | 3 +
.../org/apache/hadoop/hbase/wal/WALFactory.java | 4 +-
.../hbase/replication/TestSyncReplication.java | 200 +++
.../wal/TestSyncReplicationWALProvider.java | 8 +-
15 files changed, 401 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/010cc7d7/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index d69cf3b..891143a 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1318,9 +1318,6 @@ public final class HConstants {
public static final String NOT_IMPLEMENTED = "Not implemented";
- // TODO: need to find a better place to hold it.
- public static final String SYNC_REPLICATION_ENABLED =
"hbase.replication.sync.enabled";
-
private HConstants() {
// Can't be instantiated with this ctor.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/010cc7d7/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index c361389..fe08be5 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -385,7 +385,8 @@ enum PeerSyncReplicationStateTransitionState {
REOPEN_ALL_REGIONS_IN_PEER = 5;
TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE = 6;
REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END = 7;
- POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 8;
+ CREATE_DIR_FOR_REMOTE_WAL = 8;
+ POST_PEER_SYNC_REPLICATION_STATE_TRANSITION = 9;
}
message PeerModificationStateData {
http://git-wip-us.apache.org/repos/asf/hbase/blob/010cc7d7/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
--
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index d633be9..9d0c05d 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -37,6 +37,10 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public final class ReplicationUtils {
+ public static final String SYNC_REPLICATION_ENABLED =
"hbase.replication.sync.enabled";
+
+ public static final String REPLICATION_ATTR_NAME = "__rep__";
+
private ReplicationUtils() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/010cc7d7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
index c253bff..e53abc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replicat
[37/50] [abbrv] hbase git commit: HBASE-15151 ensure findbugs check runs in nightly tests on all branches.
HBASE-15151 ensure findbugs check runs in nightly tests on all branches.
Signed-off-by: Mike Drob
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/383b7cc5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/383b7cc5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/383b7cc5
Branch: refs/heads/HBASE-19064
Commit: 383b7cc5920b55e0b4d72cbe555089765d375dca
Parents: ddc9af9
Author: Sean Busbey
Authored: Sun Feb 25 00:35:45 2018 -0600
Committer: Sean Busbey
Committed: Thu Mar 1 15:21:40 2018 -0600
--
dev-support/Jenkinsfile | 7 ++-
1 file changed, 2 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/383b7cc5/dev-support/Jenkinsfile
--
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index fe3676d..fb38bbe 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -255,7 +255,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
-TESTS = 'mvninstall,compile,javac,unit,htmlout'
+TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
// This isn't strictly needed on branches that only support jdk8,
but doesn't hurt
@@ -327,10 +327,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
-// Failure in any stage fails the build and consecutive stages are
not built.
-// Findbugs is part of this last yetus stage to prevent findbugs
precluding hadoop3
-// tests.
-TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
+TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
// This isn't strictly needed on branches that only support jdk8,
but doesn't hurt
[29/50] [abbrv] hbase git commit: HBASE-19656 Disable TestAssignmentManagerMetrics for beta-1
HBASE-19656 Disable TestAssignmentManagerMetrics for beta-1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/372c88ab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/372c88ab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/372c88ab
Branch: refs/heads/HBASE-19064
Commit: 372c88abd6d679a12d020c880e4cc6adec9f353a
Parents: d7adc58
Author: Michael Stack
Authored: Thu Dec 28 12:15:00 2017 -0800
Committer: Michael Stack
Committed: Wed Feb 28 21:30:44 2018 -0800
--
.../apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/372c88ab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
index fb8f214..aa3a20c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerMetrics.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -44,6 +45,9 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assert.fail;
+
+@Ignore // Disabled temporarily; reenable
@Category(MediumTests.class)
public class TestAssignmentManagerMetrics {
[19/50] [abbrv] hbase git commit: HBASE-20100 TestEnableTableProcedure flakey
HBASE-20100 TestEnableTableProcedure flakey
Allow OPEN as a possible state when update region transition state.
Usually state is OPENING but if crash before finish step is completed,
on replay, master may have read that the state is OPEN from meta table
and so will think it open... When we replay the procedure finish, allow
that the region is already OPEN.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ba063abd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ba063abd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ba063abd
Branch: refs/heads/HBASE-19064
Commit: ba063abd2f4b0aaa0622a8665c4ba96ed31eafb8
Parents: f01c90b
Author: Michael Stack
Authored: Tue Feb 27 12:06:48 2018 -0800
Committer: Michael Stack
Committed: Tue Feb 27 14:51:21 2018 -0800
--
.../org/apache/hadoop/hbase/master/assignment/AssignProcedure.java | 2 +-
.../org/apache/hadoop/hbase/master/assignment/RegionStates.java| 2 ++
.../hadoop/hbase/master/assignment/RegionTransitionProcedure.java | 1 +
3 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
index 65eafe7..fd31553 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
@@ -208,7 +208,7 @@ public class AssignProcedure extends
RegionTransitionProcedure {
}
}
}
-LOG.info("Start " + this + "; " + regionNode.toShortString() +
+LOG.info("Starting " + this + "; " + regionNode.toShortString() +
"; forceNewPlan=" + this.forceNewPlan +
", retain=" + retain);
env.getAssignmentManager().queueAssign(regionNode);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
index fa94495..adeba67 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
@@ -60,6 +60,8 @@ public class RegionStates {
private static final Logger LOG =
LoggerFactory.getLogger(RegionStates.class);
protected static final State[] STATES_EXPECTED_ON_OPEN = new State[] {
+State.OPEN, // State may already be OPEN if we died after receiving the
OPEN from regionserver
+// but before complete finish of AssignProcedure. HBASE-20100.
State.OFFLINE, State.CLOSED, // disable/offline
State.SPLITTING, State.SPLIT, // ServerCrashProcedure
State.OPENING, State.FAILED_OPEN, // already in-progress (retrying)
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 8277dbe..7ce7454 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -317,6 +317,7 @@ public abstract class RegionTransitionProcedure
case REGION_TRANSITION_FINISH:
// 3. wait assignment response. completion/failure
+LOG.debug("Finishing {}; {}", this, regionNode.toShortString());
finishTransition(env, regionNode);
am.removeRegionInTransition(regionNode, this);
return null;
[21/50] [abbrv] hbase git commit: HBASE-20084 Refactor the RSRpcServices#doBatchOp
HBASE-20084 Refactor the RSRpcServices#doBatchOp
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/197bd790
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/197bd790
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/197bd790
Branch: refs/heads/HBASE-19064
Commit: 197bd790701553bd5c7de8b6af47500e0e028920
Parents: 7f6e971
Author: Chia-Ping Tsai
Authored: Mon Feb 26 20:49:05 2018 +0800
Committer: Chia-Ping Tsai
Committed: Wed Feb 28 15:15:34 2018 +0800
--
.../hbase/regionserver/RSRpcServices.java | 115 ++-
1 file changed, 58 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/197bd790/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 7e01c9a..4dd826f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -121,6 +121,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessChecker;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CollectionUtils;
import org.apache.hadoop.hbase.util.DNS;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
@@ -763,7 +764,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
// Gather up CONTIGUOUS Puts and Deletes in this mutations List. Idea is
that rather than do
// one at a time, we instead pass them in batch. Be aware that the
corresponding
// ResultOrException instance that matches each Put or Delete is then
added down in the
-// doBatchOp call. We should be staying aligned though the Put and Delete
are deferred/batched
+// doNonAtomicBatchOp call. We should be staying aligned though the Put
and Delete are
+// deferred/batched
List mutations = null;
long maxQuotaResultSize = Math.min(maxScannerResultSize,
quota.getReadAvailable());
IOException sizeIOE = null;
@@ -802,7 +804,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
// use it for the response.
//
// This will create a copy in the builder.
- hasResultOrException = true;
NameBytesPair pair = ResponseConverter.buildException(sizeIOE);
resultOrExceptionBuilder.setException(pair);
context.incrementResponseExceptionSize(pair.getSerializedSize());
@@ -829,29 +830,23 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
}
} else if (action.hasServiceCall()) {
hasResultOrException = true;
- try {
-com.google.protobuf.Message result =
-execServiceOnRegion(region, action.getServiceCall());
-ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder
=
-ClientProtos.CoprocessorServiceResult.newBuilder();
-resultOrExceptionBuilder.setServiceResult(
-serviceResultBuilder.setValue(
- serviceResultBuilder.getValueBuilder()
-.setName(result.getClass().getName())
-// TODO: Copy!!!
-
.setValue(UnsafeByteOperations.unsafeWrap(result.toByteArray();
- } catch (IOException ioe) {
-rpcServer.getMetrics().exception(ioe);
-NameBytesPair pair = ResponseConverter.buildException(ioe);
-resultOrExceptionBuilder.setException(pair);
-context.incrementResponseExceptionSize(pair.getSerializedSize());
- }
+ com.google.protobuf.Message result =
+execServiceOnRegion(region, action.getServiceCall());
+ ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder =
+ClientProtos.CoprocessorServiceResult.newBuilder();
+ resultOrExceptionBuilder.setServiceResult(
+serviceResultBuilder.setValue(
+ serviceResultBuilder.getValueBuilder()
+.setName(result.getClass().getName())
+// TODO: Copy!!!
+
.setValue(UnsafeByteOperations.unsafeWrap(result.toByteArray();
} else if (action.hasMutation()) {
MutationType type = action.getMutation().getMutateType();
if (type != MutationType.PUT && type != MutationType.DELETE &&
mutations != null &&
!mutatio
[14/50] [abbrv] hbase git commit: HBASE-20089 Use the ASF recommended naming for SHA512 xsum files
HBASE-20089 Use the ASF recommended naming for SHA512 xsum files
Signed-off-by: Andrew Purtell
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e47d1e44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e47d1e44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e47d1e44
Branch: refs/heads/HBASE-19064
Commit: e47d1e443e17408cf2131e4490397a7cee18288e
Parents: dbd8013
Author: Josh Elser
Authored: Mon Feb 26 16:30:16 2018 -0500
Committer: Josh Elser
Committed: Tue Feb 27 09:51:10 2018 -0500
--
dev-support/make_rc.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e47d1e44/dev-support/make_rc.sh
--
diff --git a/dev-support/make_rc.sh b/dev-support/make_rc.sh
index f067ee9..640bcd1 100755
--- a/dev-support/make_rc.sh
+++ b/dev-support/make_rc.sh
@@ -107,13 +107,13 @@ build_bin
MAVEN_OPTS="${mvnopts}" ${mvn} deploy -DskipTests -Papache-release -Prelease \
-Dmaven.repo.local=${output_dir}/repository
-# Do sha1 and md5
+# Do sha512 and md5
cd ${output_dir}
-for i in *.tar.gz; do echo $i; gpg --print-md SHA512 $i > $i.sha ; done
+for i in *.tar.gz; do echo $i; gpg --print-md SHA512 $i > $i.sha512 ; done
for i in *.tar.gz; do echo $i; gpg --print-md MD5 $i > $i.md5 ; done
echo "Check the content of ${output_dir}. If good, sign and push to
dist.apache.org"
echo " cd ${output_dir}"
echo ' for i in *.tar.gz; do echo $i; gpg --armor --output $i.asc --detach-sig
$i ; done'
-echo ' rsync -av ${output_dir}/*.gz ${output_dir}/*.md5 ${output_dir}/*.sha
${output_dir}/*.asc ${APACHE_HBASE_DIST_DEV_DIR}/${hbase_name}/'
+echo ' rsync -av ${output_dir}/*.gz ${output_dir}/*.md5 ${output_dir}/*.sha512
${output_dir}/*.asc ${APACHE_HBASE_DIST_DEV_DIR}/${hbase_name}/'
echo "Check the content deployed to maven. If good, close the repo and record
links of temporary staging repo"
[06/50] [abbrv] hbase git commit: HBASE-19974 Fix decommissioned servers cannot be removed by remove_servers_rsgroup methods
HBASE-19974 Fix decommissioned servers cannot be removed by
remove_servers_rsgroup methods
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a29b3caf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a29b3caf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a29b3caf
Branch: refs/heads/HBASE-19064
Commit: a29b3caf4dbc7b8833474ef5da5438f7f6907e00
Parents: 2beda62
Author: haxiaolin
Authored: Mon Feb 26 14:25:01 2018 +0800
Committer: tedyu
Committed: Mon Feb 26 07:30:56 2018 -0800
--
.../hbase/rsgroup/RSGroupAdminServer.java | 8 +++-
.../hadoop/hbase/rsgroup/TestRSGroups.java | 7 ++-
.../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 46
3 files changed, 40 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
index aba57fe..094fc1d 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
@@ -675,8 +675,12 @@ public class RSGroupAdminServer implements RSGroupAdmin {
private void checkForDeadOrOnlineServers(Set servers) throws
ConstraintException {
// This uglyness is because we only have Address, not ServerName.
Set onlineServers = new HashSet<>();
-for(ServerName server:
master.getServerManager().getOnlineServers().keySet()) {
- onlineServers.add(server.getAddress());
+List drainingServers =
master.getServerManager().getDrainingServersList();
+for (ServerName server :
master.getServerManager().getOnlineServers().keySet()) {
+ // Only online but not decommissioned servers are really online
+ if (!drainingServers.contains(server)) {
+onlineServers.add(server.getAddress());
+ }
}
Set deadServers = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
index 9116f3b..610278a 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.net.Address;
@@ -66,7 +65,6 @@ public class TestRSGroups extends TestRSGroupsBase {
HBaseClassTestRule.forClass(TestRSGroups.class);
protected static final Logger LOG =
LoggerFactory.getLogger(TestRSGroups.class);
- private static HMaster master;
private static boolean INIT = false;
private static RSGroupAdminEndpoint rsGroupAdminEndpoint;
@@ -126,6 +124,11 @@ public class TestRSGroups extends TestRSGroupsBase {
deleteNamespaceIfNecessary();
deleteGroups();
+for(ServerName sn : admin.listDecommissionedRegionServers()){
+ admin.recommissionRegionServer(sn, null);
+}
+assertTrue(admin.listDecommissionedRegionServers().isEmpty());
+
int missing = NUM_SLAVES_BASE - getNumServers();
LOG.info("Restoring servers: "+missing);
for(int i=0; ihttp://git-wip-us.apache.org/repos/asf/hbase/blob/a29b3caf/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index c3f7eef..76bcd20 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -25,8 +25,10 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.security.SecureRandom;
+import java.util.ArrayList;
import java.util.EnumSet;
i
[11/50] [abbrv] hbase git commit: HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address review
HBASE-20069 fix existing findbugs errors in hbase-server; ADDENDUM Address
review
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d272ac90
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d272ac90
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d272ac90
Branch: refs/heads/HBASE-19064
Commit: d272ac908ceb4696e05431066ae02d953fa6fc9d
Parents: d3aefe7
Author: Michael Stack
Authored: Mon Feb 26 16:26:36 2018 -0800
Committer: Michael Stack
Committed: Mon Feb 26 23:16:21 2018 -0800
--
.../hadoop/hbase/master/ClusterStatusPublisher.java | 6 +-
.../hadoop/hbase/regionserver/MemStoreFlusher.java| 14 +++---
.../hbase/regionserver/RegionCoprocessorHost.java | 4 +++-
3 files changed, 11 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 5e97204..21fa263 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -134,7 +134,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
@Override
protected void chore() {
-if (!connected) {
+if (!isConnected()) {
return;
}
@@ -170,6 +170,10 @@ public class ClusterStatusPublisher extends ScheduledChore
{
publisher.close();
}
+ private synchronized boolean isConnected() {
+return this.connected;
+ }
+
/**
* Create the dead server to send. A dead server is sent NB_SEND times. We
send at max
* MAX_SERVER_PER_MESSAGE at a time. if there are too many dead servers, we
send the newly
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index a0e65ec..23321e8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -90,17 +90,9 @@ class MemStoreFlusher implements FlushRequester {
private FlushType flushType;
/**
- * Singleton instance of this class inserted into flush queue.
+ * Singleton instance inserted into flush queue used for signaling.
*/
- private static final WakeupFlushThread WAKEUPFLUSH_INSTANCE = new
WakeupFlushThread();
-
- /**
- * Marker class used as a token inserted into flush queue that ensures the
flusher does not sleep.
- * Create a single instance only.
- */
- private static final class WakeupFlushThread implements FlushQueueEntry {
-private WakeupFlushThread() {}
-
+ private static final FlushQueueEntry WAKEUPFLUSH_INSTANCE = new
FlushQueueEntry() {
@Override
public long getDelay(TimeUnit unit) {
return 0;
@@ -120,7 +112,7 @@ class MemStoreFlusher implements FlushRequester {
public int hashCode() {
return 42;
}
- }
+ };
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/d272ac90/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index f3c93dc..47b389a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1119,6 +1119,8 @@ public class RegionCoprocessorHost
* @return true or false to return to client if default processing should be
bypassed, or null
* otherwise
*/
+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_BOOLEAN_RETURN_NULL",
+ justification="Null is legit")
public Boolean preCheckAndPutAfterRowLock(
final byte[] row, final byte[] family, final byte[] qualifier, final
CompareOperator op,
final ByteArrayComparable comparator, final Put put) throws IOException {
@@ -1207,7 +1209,7 @@ public class RegionCoprocessorHost
* or null otherwise
*/
@e
[35/50] [abbrv] hbase git commit: HBASE-20055 Removed declaration of un-thrown exceptions and unused setRegionStateBackToOpen() from MergeTableRegionsProcedure
HBASE-20055 Removed declaration of un-thrown exceptions and unused
setRegionStateBackToOpen() from MergeTableRegionsProcedure
Plus some minor cleanup.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e86ffedb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e86ffedb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e86ffedb
Branch: refs/heads/HBASE-19064
Commit: e86ffedb453afd0554132748ce74a47817e5c2f4
Parents: ee1f26c
Author: Umesh Agashe
Authored: Thu Feb 22 16:29:52 2018 -0800
Committer: Michael Stack
Committed: Thu Mar 1 08:46:23 2018 -0800
--
.../assignment/MergeTableRegionsProcedure.java | 46 +---
.../TestRegionMergeTransactionOnCluster.java| 2 +-
2 files changed, 22 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e86ffedb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 7c041e7..052ba7f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -52,8 +52,6 @@ import
org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
-import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
-import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
@@ -146,7 +144,7 @@ public class MergeTableRegionsProcedure
}
if (!RegionInfo.areAdjacent(regionToMergeA, regionToMergeB)) {
- String msg = "Unable to merge not adjacent regions " +
regionToMergeA.getShortNameToLog() +
+ String msg = "Unable to merge non-adjacent regions " +
regionToMergeA.getShortNameToLog() +
", " + regionToMergeB.getShortNameToLog() + " where forcible = " +
forcible;
LOG.warn(msg);
if (!forcible) {
@@ -194,7 +192,7 @@ public class MergeTableRegionsProcedure
private static long getMergedRegionIdTimestamp(final RegionInfo
regionToMergeA,
final RegionInfo regionToMergeB) {
long rid = EnvironmentEdgeManager.currentTime();
-// Regionid is timestamp. Merged region's id can't be less than that of
+// Region Id is a timestamp. Merged region's id can't be less than that of
// merging regions else will insert at wrong location in hbase:meta (See
HBASE-710).
if (rid < regionToMergeA.getRegionId() || rid <
regionToMergeB.getRegionId()) {
LOG.warn("Clock skew; merging regions id are " +
regionToMergeA.getRegionId()
@@ -205,8 +203,8 @@ public class MergeTableRegionsProcedure
}
@Override
- protected Flow executeFromState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
- throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
+ protected Flow executeFromState(final MasterProcedureEnv env,
+ final MergeTableRegionsState state) {
LOG.trace("{} execute state={}", this, state);
try {
switch (state) {
@@ -260,7 +258,7 @@ public class MergeTableRegionsProcedure
RegionInfo.getShortNameToLog(regionsToMerge) + " in the table " +
getTableName() +
" (in state=" + state + ")";
if (!isRollbackSupported(state)) {
-// We reach a state that cannot be rolled back. We just need to keep
retry.
+// We reach a state that cannot be rolled back. We just need to keep
retrying.
LOG.warn(msg, e);
} else {
LOG.error(msg, e);
@@ -277,9 +275,8 @@ public class MergeTableRegionsProcedure
* See HBASE-19851 for details.
*/
@Override
- protected void rollbackState(
- final MasterProcedureEnv env,
- final MergeTableRegionsState state) throws IOException,
InterruptedException {
+ protected void rollbackState(final MasterProcedureEnv env, final
MergeTableRegionsState state)
+ throws IOException {
if (isTraceEnabled())
