[hbase] branch HBASE-21512 updated (b2dc796 -> 19c9908)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git. discard b2dc796 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection discard fd3e534 HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint discard fa8c63a HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin discard 53a9645 HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager discard 009a31b HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer add fbf7937 HBASE-21663 Add replica scan support new f9914c5 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer new ebf6f9c HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager new b542ca8 HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin new 314094b HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint new 19c9908 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (b2dc796) \ N -- N -- N refs/heads/HBASE-21512 (19c9908) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../client/AsyncAdminRequestRetryingCaller.java| 15 +- .../hbase/client/AsyncBatchRpcRetryingCaller.java | 6 +- .../hadoop/hbase/client/AsyncClientScanner.java| 103 +++- .../hbase/client/AsyncConnectionConfiguration.java | 21 +++ .../hadoop/hbase/client/AsyncConnectionImpl.java | 5 +- .../AsyncMasterRequestRpcRetryingCaller.java | 16 +- .../hbase/client/AsyncRpcRetryingCaller.java | 15 +- .../client/AsyncRpcRetryingCallerFactory.java | 6 +- .../AsyncScanSingleRegionRpcRetryingCaller.java| 9 +- .../AsyncServerRequestRpcRetryingCaller.java | 16 +- .../AsyncSingleRequestRpcRetryingCaller.java | 10 +- .../hbase/client/ConnectionConfiguration.java | 9 +- .../hadoop/hbase/client/ConnectionUtils.java | 101 ++- .../hadoop/hbase/client/RawAsyncTableImpl.java | 93 ++ ... AbstractTestAsyncTableRegionReplicasRead.java} | 102 +-- .../client/TestAsyncTableRegionReplicasGet.java| 187 + .../client/TestAsyncTableRegionReplicasScan.java | 76 + 17 files changed, 370 insertions(+), 420 deletions(-) copy hbase-server/src/test/java/org/apache/hadoop/hbase/client/{TestAsyncTableRegionReplicasGet.java => AbstractTestAsyncTableRegionReplicasRead.java} (72%) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRegionReplicasScan.java
[hbase] 02/05: HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit ebf6f9cc013a418f65a69ec472de1522aa1d33ff Author: zhangduo AuthorDate: Sat Dec 1 21:15:48 2018 +0800 HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager --- .../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 5 +- .../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 ++- .../hadoop/hbase/regionserver/HRegionServer.java | 2 +- .../hbase/regionserver/SecureBulkLoadManager.java | 24 - .../hadoop/hbase/security/token/TokenUtil.java | 57 +- .../hadoop/hbase/security/token/TestTokenUtil.java | 42 6 files changed, 96 insertions(+), 41 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index a3d49b5..d9e620b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -261,13 +261,12 @@ public final class ProtobufUtil { * just {@link ServiceException}. Prefer this method to * {@link #getRemoteException(ServiceException)} because trying to * contain direct protobuf references. - * @param e */ - public static IOException handleRemoteException(Exception e) { + public static IOException handleRemoteException(Throwable e) { return makeIOExceptionOfException(e); } - private static IOException makeIOExceptionOfException(Exception e) { + private static IOException makeIOExceptionOfException(Throwable e) { Throwable t = e; if (e instanceof ServiceException || e instanceof org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index fea81f1..de2fb7d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ByteBufferExtendedCell; @@ -123,6 +122,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.Service; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; + import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; @@ -343,13 +343,12 @@ public final class ProtobufUtil { * just {@link ServiceException}. Prefer this method to * {@link #getRemoteException(ServiceException)} because trying to * contain direct protobuf references. - * @param e */ - public static IOException handleRemoteException(Exception e) { + public static IOException handleRemoteException(Throwable e) { return makeIOExceptionOfException(e); } - private static IOException makeIOExceptionOfException(Exception e) { + private static IOException makeIOExceptionOfException(Throwable e) { Throwable t = e; if (e instanceof ServiceException) { t = e.getCause(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index b4b1d3e..bbc9d3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1937,7 +1937,7 @@ public class HRegionServer extends HasThread implements if (!isStopped() && !isAborted()) { initializeThreads(); } -this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, clusterConnection); +this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, asyncClusterConnection); this.secureBulkLoadManager.start(); // Health checker thread. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index 566a6b6..add6519 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBul
[hbase] 04/05: HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 314094b9c32786094bf6972cd849afc587ffdd6b Author: zhangduo AuthorDate: Tue Jan 1 21:27:14 2019 +0800 HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint --- .../hbase/client/AsyncRegionServerAdmin.java | 14 +--- .../hbase/protobuf/ReplicationProtbufUtil.java | 35 ++- .../HBaseInterClusterReplicationEndpoint.java | 31 + .../regionserver/ReplicationSinkManager.java | 40 -- .../hbase/replication/SyncReplicationTestBase.java | 12 +++ .../regionserver/TestReplicationSinkManager.java | 21 +--- 6 files changed, 74 insertions(+), 79 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java index 9accd89..b9141a9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.yetus.audience.InterfaceAudience; @@ -94,9 +95,9 @@ public class AsyncRegionServerAdmin { void call(AdminService.Interface stub, HBaseRpcController controller, RpcCallback done); } - private CompletableFuture call(RpcCall rpcCall) { + private CompletableFuture call(RpcCall rpcCall, CellScanner cellScanner) { CompletableFuture future = new CompletableFuture<>(); -HBaseRpcController controller = conn.rpcControllerFactory.newController(); +HBaseRpcController controller = conn.rpcControllerFactory.newController(cellScanner); try { rpcCall.call(conn.getAdminStub(server), controller, new RpcCallback() { @@ -115,6 +116,10 @@ public class AsyncRegionServerAdmin { return future; } + private CompletableFuture call(RpcCall rpcCall) { +return call(rpcCall, null); + } + public CompletableFuture getRegionInfo(GetRegionInfoRequest request) { return call((stub, controller, done) -> stub.getRegionInfo(controller, request, done)); } @@ -154,8 +159,9 @@ public class AsyncRegionServerAdmin { } public CompletableFuture replicateWALEntry( - ReplicateWALEntryRequest request) { -return call((stub, controller, done) -> stub.replicateWALEntry(controller, request, done)); + ReplicateWALEntryRequest request, CellScanner cellScanner) { +return call((stub, controller, done) -> stub.replicateWALEntry(controller, request, done), + cellScanner); } public CompletableFuture replay(ReplicateWALEntryRequest request) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java index c1b3911..74fad26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java @@ -20,51 +20,54 @@ package org.apache.hadoop.hbase.protobuf; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin; import org.apache.hadoop.hbase.io.SizedCellScanner; -import org.apache.hadoop.hbase.ipc.HBaseRpcController; -import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl; import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.base.Throwables; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; @InterfaceAudience.Private public class ReplicationProtbufUtil { + /** - * A helper to replicate a list of WAL entries using admin protocol. - * @param admin Admin service + * A helper to r
[hbase] 03/05: HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit b542ca810680409abda91c733e2d3fed91e4c622 Author: zhangduo AuthorDate: Thu Dec 6 21:25:34 2018 +0800 HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin --- .../hbase/client/AsyncClusterConnection.java | 6 + .../hadoop/hbase/client/AsyncConnectionImpl.java | 4 + .../hbase/client/AsyncRegionServerAdmin.java | 210 + .../org/apache/hadoop/hbase/util/FutureUtils.java | 2 +- .../org/apache/hadoop/hbase/master/HMaster.java| 15 +- .../apache/hadoop/hbase/master/ServerManager.java | 67 --- .../master/procedure/RSProcedureDispatcher.java| 44 +++-- 7 files changed, 262 insertions(+), 86 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index c7dea25..1327fd7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.yetus.audience.InterfaceAudience; @@ -27,6 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience; public interface AsyncClusterConnection extends AsyncConnection { /** + * Get the admin service for the given region server. + */ + AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName); + + /** * Get the nonce generator for this connection. */ NonceGenerator getNonceGenerator(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 50e27c4..9bead83 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -359,4 +359,8 @@ class AsyncConnectionImpl implements AsyncClusterConnection { return new HBaseHbck(MasterProtos.HbckService.newBlockingStub( rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)), rpcControllerFactory); } + + public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { +return new AsyncRegionServerAdmin(serverName, this); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java new file mode 100644 index 000..9accd89 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ipc.HBaseRpcController; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.ge
[hbase] 01/05: HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit f9914c55419c97eac4616253e7fb5b90a8e3ab8f Author: zhangduo AuthorDate: Fri Nov 30 08:23:47 2018 +0800 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer --- .../hbase/client/AsyncClusterConnection.java | 38 + .../hadoop/hbase/client/AsyncConnectionImpl.java | 14 +++-- .../hbase/client/ClusterConnectionFactory.java | 63 ++ .../hadoop/hbase/client/ConnectionFactory.java | 5 +- .../apache/hadoop/hbase/util/ReflectionUtils.java | 22 .../main/java/org/apache/hadoop/hbase/Server.java | 20 +++ .../org/apache/hadoop/hbase/master/HMaster.java| 3 ++ .../hadoop/hbase/regionserver/HRegionServer.java | 56 +-- .../regionserver/ReplicationSyncUp.java| 6 +++ .../hadoop/hbase/MockRegionServerServices.java | 5 ++ .../client/TestAsyncNonMetaRegionLocator.java | 2 +- ...stAsyncNonMetaRegionLocatorConcurrenyLimit.java | 2 +- .../client/TestAsyncRegionLocatorTimeout.java | 2 +- .../TestAsyncSingleRequestRpcRetryingCaller.java | 4 +- .../hbase/client/TestAsyncTableNoncedRetry.java| 2 +- .../hbase/master/MockNoopMasterServices.java | 6 +++ .../hadoop/hbase/master/MockRegionServer.java | 5 ++ .../hbase/master/TestActiveMasterManager.java | 6 +++ .../hbase/master/cleaner/TestHFileCleaner.java | 6 +++ .../hbase/master/cleaner/TestHFileLinkCleaner.java | 6 +++ .../hbase/master/cleaner/TestLogsCleaner.java | 6 +++ .../cleaner/TestReplicationHFileCleaner.java | 6 +++ .../hbase/regionserver/TestHeapMemoryManager.java | 6 +++ .../hbase/regionserver/TestSplitLogWorker.java | 6 +++ .../hadoop/hbase/regionserver/TestWALLockup.java | 6 +++ .../replication/TestReplicationTrackerZKImpl.java | 6 +++ .../regionserver/TestReplicationSourceManager.java | 6 +++ .../security/token/TestTokenAuthentication.java| 6 +++ .../org/apache/hadoop/hbase/util/MockServer.java | 6 +++ 29 files changed, 290 insertions(+), 37 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java new file mode 100644 index 000..c7dea25 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.ipc.RpcClient; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * The asynchronous connection for internal usage. + */ +@InterfaceAudience.Private +public interface AsyncClusterConnection extends AsyncConnection { + + /** + * Get the nonce generator for this connection. + */ + NonceGenerator getNonceGenerator(); + + /** + * Get the rpc client we used to communicate with other servers. + */ + RpcClient getRpcClient(); +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 3cbd950..50e27c4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.client.NonceGenerator.CLIENT_NONCES_ENABLE import static org.apache.hadoop.hbase.util.FutureUtils.addListener; import java.io.IOException; +import java.net.SocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -63,7 +64,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ * The implementation of AsyncConnection. */ @InterfaceAudience.Private -class AsyncConnectionImpl implements AsyncConnection { +class AsyncConnectionImpl implements AsyncClusterConnecti
[hbase] 05/05: HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 19c99087ed5742acabf7d649ef9fc558e9ff3619 Author: Duo Zhang AuthorDate: Wed Dec 12 09:33:33 2018 +0800 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection --- .../hbase/client/AsyncClusterConnection.java | 8 ++ .../hadoop/hbase/client/AsyncConnectionImpl.java | 8 ++ .../hbase/client/ClusterConnectionFactory.java | 16 +-- .../hadoop/hbase/client/RawAsyncHBaseAdmin.java| 36 --- .../org/apache/hadoop/hbase/util/FutureUtils.java | 22 + .../master/procedure/RSProcedureDispatcher.java| 34 +-- .../hbase/protobuf/ReplicationProtbufUtil.java | 15 +-- .../hadoop/hbase/regionserver/HRegionServer.java | 3 +- .../handler/RegionReplicaFlushHandler.java | 110 - 9 files changed, 132 insertions(+), 120 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index 1327fd7..f1f64ca 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hbase.client; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; + /** * The asynchronous connection for internal usage. */ @@ -41,4 +44,9 @@ public interface AsyncClusterConnection extends AsyncConnection { * Get the rpc client we used to communicate with other servers. */ RpcClient getRpcClient(); + + /** + * Flush a region and get the response. + */ + CompletableFuture flush(byte[] regionName, boolean writeFlushWALMarker); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 9bead83..ce6bfac 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -55,6 +55,7 @@ import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse; @@ -363,4 +364,11 @@ class AsyncConnectionImpl implements AsyncClusterConnection { public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { return new AsyncRegionServerAdmin(serverName, this); } + + @Override + public CompletableFuture flush(byte[] regionName, + boolean writeFlushWALMarker) { +RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin(); +return admin.flushRegionInternal(regionName, writeFlushWALMarker); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java index 68c0630..79484db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java @@ -18,15 +18,12 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; -import java.io.InterruptedIOException; import java.net.SocketAddress; -import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.com.google.common.base.Throwables; - /** * The factory for creating {@link AsyncClusterConnection}. */ @@ -48,16 +45,7 @@ public final class ClusterConnectionFactory { public static AsyncClusterConnection createAsyncClusterConnection(Configuration conf, SocketAddress localAddress, User user) throws IOException { AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf); -String clusterId; -try { - clusterId = registry.getClusterId().get(); -} catch (InterruptedException
[hbase-site] branch asf-site updated: this should hit pubsub and mailing list, finally!
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 0abdb46 this should hit pubsub and mailing list, finally! 0abdb46 is described below commit 0abdb461aa43f028db81703af54bf554fcec0477 Author: jenkins AuthorDate: Fri Jan 11 09:22:19 2019 + this should hit pubsub and mailing list, finally! --- index.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/index.html b/index.html index df9bbc4..ff755de 100644 --- a/index.html +++ b/index.html @@ -435,5 +435,3 @@ Apache HBase is an open-source, distributed, versioned, non-relational database - -
[hbase] branch HBASE-21512 updated: HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/HBASE-21512 by this push: new d6fbe51 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection d6fbe51 is described below commit d6fbe513638fa66b578104825b77b047231ae31e Author: Duo Zhang AuthorDate: Fri Jan 11 16:22:24 2019 +0800 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection --- .../hadoop/hbase/client/AsyncConnectionImpl.java | 24 +- .../hbase/client/AsyncClusterConnection.java | 17 + .../hbase/client/AsyncClusterConnectionImpl.java | 80 +++ .../AsyncRegionReplicaReplayRetryingCaller.java| 146 .../hbase/client/AsyncRegionServerAdmin.java | 5 +- .../hbase/client/ClusterConnectionFactory.java | 2 +- .../hbase/protobuf/ReplicationProtbufUtil.java | 31 +- .../handler/RegionReplicaFlushHandler.java | 3 +- .../hbase/replication/ReplicationEndpoint.java | 35 +- .../RegionReplicaReplicationEndpoint.java | 783 +++-- .../regionserver/ReplicationSource.java| 2 +- .../hbase/client/TestAsyncTableNoncedRetry.java| 2 +- .../TestRegionReplicaReplicationEndpoint.java | 56 +- ...stRegionReplicaReplicationEndpointNoMaster.java | 99 ++- 14 files changed, 627 insertions(+), 658 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index ce6bfac..f6a2149 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -55,7 +55,6 @@ import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse; @@ -65,7 +64,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ * The implementation of AsyncConnection. */ @InterfaceAudience.Private -class AsyncConnectionImpl implements AsyncClusterConnection { +class AsyncConnectionImpl implements AsyncConnection { private static final Logger LOG = LoggerFactory.getLogger(AsyncConnectionImpl.class); @@ -85,7 +84,7 @@ class AsyncConnectionImpl implements AsyncClusterConnection { private final int rpcTimeout; - private final RpcClient rpcClient; + protected final RpcClient rpcClient; final RpcControllerFactory rpcControllerFactory; @@ -160,16 +159,10 @@ class AsyncConnectionImpl implements AsyncClusterConnection { } // ditto - @Override - public NonceGenerator getNonceGenerator() { + NonceGenerator getNonceGenerator() { return nonceGenerator; } - @Override - public RpcClient getRpcClient() { -return rpcClient; - } - private ClientService.Interface createRegionServerStub(ServerName serverName) throws IOException { return ClientService.newStub(rpcClient.createRpcChannel(serverName, user, rpcTimeout)); } @@ -360,15 +353,4 @@ class AsyncConnectionImpl implements AsyncClusterConnection { return new HBaseHbck(MasterProtos.HbckService.newBlockingStub( rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)), rpcControllerFactory); } - - public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { -return new AsyncRegionServerAdmin(serverName, this); - } - - @Override - public CompletableFuture flush(byte[] regionName, - boolean writeFlushWALMarker) { -RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin(); -return admin.flushRegionInternal(regionName, writeFlushWALMarker); - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java similarity index 72% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index f1f64ca..0ad77ba 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hbase.client;
[hbase] branch HBASE-20112 deleted (was 3a12912)
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a change to branch HBASE-20112 in repository https://gitbox.apache.org/repos/asf/hbase.git. was 3a12912 HBASE-20112 register nightly junit over hadoop3 results with jenkins. This change permanently discards the following revisions: discard 3a12912 HBASE-20112 register nightly junit over hadoop3 results with jenkins.
[hbase] branch HBASE-20364 deleted (was da8ede8)
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a change to branch HBASE-20364 in repository https://gitbox.apache.org/repos/asf/hbase.git. was da8ede8 WIP fail stages that do an scm checkout. This change permanently discards the following revisions: discard da8ede8 WIP fail stages that do an scm checkout. discard 7d04d37 HBASE-20364 ensure jira comment from nightly reflects results of run comment comes from.
[hbase] branch master updated: HBASE-21295 Update compatibility matrices
This is an automated email from the ASF dual-hosted git repository. psomogyi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 2b5b190 HBASE-21295 Update compatibility matrices 2b5b190 is described below commit 2b5b190882daeb0f9f0cf5361df3525db7119bea Author: Peter Somogyi AuthorDate: Wed Jan 9 13:59:30 2019 +0100 HBASE-21295 Update compatibility matrices Signed-off-by: Sean Busbey --- src/main/asciidoc/_chapters/configuration.adoc | 75 ++ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc index b969c8b..545cefa 100644 --- a/src/main/asciidoc/_chapters/configuration.adoc +++ b/src/main/asciidoc/_chapters/configuration.adoc @@ -93,7 +93,10 @@ This section lists required services and some required system configuration. [[java]] .Java -The following table summarizes the recommendation of the HBase community wrt deploying on various Java versions. An entry of "yes" is meant to indicate a base level of testing and willingness to help diagnose and address issues you might run into. Similarly, an entry of "no" or "Not Supported" generally means that should you run into an issue the community is likely to ask you to change the Java environment before proceeding to help. In some cases, specific guidance on limitations (e.g. [...] +The following table summarizes the recommendation of the HBase community wrt deploying on various Java versions. +A icon:check-circle[role="green"] symbol is meant to indicate a base level of testing and willingness to help diagnose and address issues you might run into. +Similarly, an entry of icon:exclamation-circle[role="yellow"] or icon:times-circle[role="red"] generally means that should you run into an issue the community is likely to ask you to change the Java environment before proceeding to help. +In some cases, specific guidance on limitations (e.g. whether compiling / unit tests work, specific operational issues, etc) will also be noted. .Long Term Support JDKs are recommended [TIP] @@ -102,32 +105,34 @@ HBase recommends downstream users rely on JDK releases that are marked as Long T .Java support by release line -[cols="1,1,1,1,1", options="header"] +[cols="6*^.^", options="header"] |=== |HBase Version |JDK 7 |JDK 8 -|JDK 9 -|JDK 10 +|JDK 9 (Non-LTS) +|JDK 10 (Non-LTS) +|JDK 11 -|2.0 +|2.0+ |icon:times-circle[role="red"] |icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] - -|1.3 -|icon:check-circle[role="green"] -|icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] - - -|1.2 +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-21110[HBASE-21110] + +|1.2+ |icon:check-circle[role="green"] |icon:check-circle[role="green"] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] -|icon:exclamation-circle[role="yellow"] Non-LTS, see link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-20264[HBASE-20264] +v|icon:exclamation-circle[role="yellow"] +link:https://issues.apache.org/jira/browse/HBASE-21110[HBASE-21110] |=== @@ -217,24 +222,24 @@ Use the following legend to interpret this table: * icon:times-circle[role="red"] = Known to not be fully-functional * icon:exclamation-circle[role="yellow"] = Not tested, may/may-not function -[cols="1,1,1,1,1,1", options="header"] +[cols="1,4*^.^", options="header"] |=== -| | HBase-1.2.x | HBase-1.3.x | HBase-1.5.x | HBase-2.0.x | HBase-2.1.x -|Hadoop-2.4.x | icon:check-circle[role="green"] | icon:check-circle[role="green"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] -|Hadoop-2.5.x | icon:check-circle[role="green"] | icon:check-circle[role="green"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] | icon:times-circle[role="red"] -|Hadoop-2.6.0 | icon:times-circle[role="re
[hbase] branch master updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 3d2580c HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect 3d2580c is described below commit 3d2580cd6dbe32497543207ae616a9559de94739 Author: Duo Zhang AuthorDate: Fri Jan 11 17:45:12 2019 +0800 HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect --- .../hadoop/hbase/DistributedHBaseCluster.java | 47 +- .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java index 5ec9e25..30a3db9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -25,20 +25,19 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.TreeSet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterManager.ServiceType; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; @@ -312,20 +311,16 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { +byte[] startKey = RegionInfo.getStartKey(regionName); HRegionLocation regionLoc = null; try (RegionLocator locator = connection.getRegionLocator(tn)) { - regionLoc = locator.getRegionLocation(regionName, true); + regionLoc = locator.getRegionLocation(startKey, true); } if (regionLoc == null) { - LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + -", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); + LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName)); return null; } - -AdminProtos.AdminService.BlockingInterface client = - ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); -ServerInfo info = ProtobufUtil.getServerInfo(null, client); -return ProtobufUtil.toServerName(info.getServerName()); +return regionLoc.getServerName(); } @Override @@ -366,17 +361,15 @@ public class DistributedHBaseCluster extends HBaseCluster { //check whether current master has changed final ServerName initMaster = initial.getMasterName(); if (!ServerName.isSameAddress(initMaster, current.getMasterName())) { - LOG.info("Restoring cluster - Initial active master : " - + initMaster.getHostAndPort() - + " has changed to : " - + current.getMasterName().getHostAndPort()); + LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() + +" has changed to : " + current.getMasterName().getAddress()); // If initial master is stopped, start it, before restoring the state. // It will come up as a backup master, if there is already an active master. try { if (!clusterManager.isRunning(ServiceType.HBASE_MASTER, initMaster.getHostname(), initMaster.getPort())) { LOG.info("Restoring cluster - starting initial active master at:" - + initMaster.getHostAndPort()); + + initMaster.getAddress()); startMaster(initMaster.getHostname(), initMaster.getPort()); } @@ -407,7 +400,7 @@ public class DistributedHBaseCluster extends HBaseCluster { backup.getHostname(), backup.getPort())) { LOG.info("Restoring cluster - starting initial backup master: " -+ backup.getHostAndPort()); ++ backup.getAddress()); startMaster(backup.getHostname(), backup.getPort()); }
[hbase] branch branch-2 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 9ec84c2 HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect 9ec84c2 is described below commit 9ec84c235f135d40dfdcfb0a629648772b88fb8e Author: Duo Zhang AuthorDate: Fri Jan 11 17:45:12 2019 +0800 HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect --- .../hadoop/hbase/DistributedHBaseCluster.java | 47 +- .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java index 5ec9e25..30a3db9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -25,20 +25,19 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.TreeSet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterManager.ServiceType; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; @@ -312,20 +311,16 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { +byte[] startKey = RegionInfo.getStartKey(regionName); HRegionLocation regionLoc = null; try (RegionLocator locator = connection.getRegionLocator(tn)) { - regionLoc = locator.getRegionLocation(regionName, true); + regionLoc = locator.getRegionLocation(startKey, true); } if (regionLoc == null) { - LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + -", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); + LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName)); return null; } - -AdminProtos.AdminService.BlockingInterface client = - ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); -ServerInfo info = ProtobufUtil.getServerInfo(null, client); -return ProtobufUtil.toServerName(info.getServerName()); +return regionLoc.getServerName(); } @Override @@ -366,17 +361,15 @@ public class DistributedHBaseCluster extends HBaseCluster { //check whether current master has changed final ServerName initMaster = initial.getMasterName(); if (!ServerName.isSameAddress(initMaster, current.getMasterName())) { - LOG.info("Restoring cluster - Initial active master : " - + initMaster.getHostAndPort() - + " has changed to : " - + current.getMasterName().getHostAndPort()); + LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() + +" has changed to : " + current.getMasterName().getAddress()); // If initial master is stopped, start it, before restoring the state. // It will come up as a backup master, if there is already an active master. try { if (!clusterManager.isRunning(ServiceType.HBASE_MASTER, initMaster.getHostname(), initMaster.getPort())) { LOG.info("Restoring cluster - starting initial active master at:" - + initMaster.getHostAndPort()); + + initMaster.getAddress()); startMaster(initMaster.getHostname(), initMaster.getPort()); } @@ -407,7 +400,7 @@ public class DistributedHBaseCluster extends HBaseCluster { backup.getHostname(), backup.getPort())) { LOG.info("Restoring cluster - starting initial backup master: " -+ backup.getHostAndPort()); ++ backup.getAddress()); startMaster(backup.getHostname(), backup.getPort());
[hbase] branch branch-2.1 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new d7db78b HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect d7db78b is described below commit d7db78b74ca586536cf73894ea6779716132e26f Author: Duo Zhang AuthorDate: Fri Jan 11 17:45:12 2019 +0800 HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect --- .../hadoop/hbase/DistributedHBaseCluster.java | 45 -- .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java index 943f2a6..1e37d24 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -31,14 +31,13 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; @@ -281,20 +280,16 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { +byte[] startKey = RegionInfo.getStartKey(regionName); HRegionLocation regionLoc = null; try (RegionLocator locator = connection.getRegionLocator(tn)) { - regionLoc = locator.getRegionLocation(regionName, true); + regionLoc = locator.getRegionLocation(startKey, true); } if (regionLoc == null) { - LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + -", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); + LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName)); return null; } - -AdminProtos.AdminService.BlockingInterface client = - ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); -ServerInfo info = ProtobufUtil.getServerInfo(null, client); -return ProtobufUtil.toServerName(info.getServerName()); +return regionLoc.getServerName(); } @Override @@ -335,17 +330,15 @@ public class DistributedHBaseCluster extends HBaseCluster { //check whether current master has changed final ServerName initMaster = initial.getMasterName(); if (!ServerName.isSameAddress(initMaster, current.getMasterName())) { - LOG.info("Restoring cluster - Initial active master : " - + initMaster.getHostAndPort() - + " has changed to : " - + current.getMasterName().getHostAndPort()); + LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() + +" has changed to : " + current.getMasterName().getAddress()); // If initial master is stopped, start it, before restoring the state. // It will come up as a backup master, if there is already an active master. try { if (!clusterManager.isRunning(ServiceType.HBASE_MASTER, initMaster.getHostname(), initMaster.getPort())) { LOG.info("Restoring cluster - starting initial active master at:" - + initMaster.getHostAndPort()); + + initMaster.getAddress()); startMaster(initMaster.getHostname(), initMaster.getPort()); } @@ -376,7 +369,7 @@ public class DistributedHBaseCluster extends HBaseCluster { backup.getHostname(), backup.getPort())) { LOG.info("Restoring cluster - starting initial backup master: " -+ backup.getHostAndPort()); ++ backup.getAddress()); startMaster(backup.getHostname(), backup.getPort()); } } catch (IOException ex) { @@ -400,7 +393,7 @@ public class DistributedHBaseCluster extends HBaseCluster { for (ServerName sn:toStart) { try { if(!clusterManager.is
[hbase] branch branch-2.0 updated: HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.0 by this push: new a8905cf HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect a8905cf is described below commit a8905cfffae7ef7707aad0351b35b9dbbcdb9e44 Author: Duo Zhang AuthorDate: Fri Jan 11 17:45:12 2019 +0800 HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect --- .../hadoop/hbase/DistributedHBaseCluster.java | 45 -- .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 ++- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java index 943f2a6..1e37d24 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -31,14 +31,13 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; @@ -281,20 +280,16 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { +byte[] startKey = RegionInfo.getStartKey(regionName); HRegionLocation regionLoc = null; try (RegionLocator locator = connection.getRegionLocator(tn)) { - regionLoc = locator.getRegionLocation(regionName, true); + regionLoc = locator.getRegionLocation(startKey, true); } if (regionLoc == null) { - LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + -", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); + LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName)); return null; } - -AdminProtos.AdminService.BlockingInterface client = - ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); -ServerInfo info = ProtobufUtil.getServerInfo(null, client); -return ProtobufUtil.toServerName(info.getServerName()); +return regionLoc.getServerName(); } @Override @@ -335,17 +330,15 @@ public class DistributedHBaseCluster extends HBaseCluster { //check whether current master has changed final ServerName initMaster = initial.getMasterName(); if (!ServerName.isSameAddress(initMaster, current.getMasterName())) { - LOG.info("Restoring cluster - Initial active master : " - + initMaster.getHostAndPort() - + " has changed to : " - + current.getMasterName().getHostAndPort()); + LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() + +" has changed to : " + current.getMasterName().getAddress()); // If initial master is stopped, start it, before restoring the state. // It will come up as a backup master, if there is already an active master. try { if (!clusterManager.isRunning(ServiceType.HBASE_MASTER, initMaster.getHostname(), initMaster.getPort())) { LOG.info("Restoring cluster - starting initial active master at:" - + initMaster.getHostAndPort()); + + initMaster.getAddress()); startMaster(initMaster.getHostname(), initMaster.getPort()); } @@ -376,7 +369,7 @@ public class DistributedHBaseCluster extends HBaseCluster { backup.getHostname(), backup.getPort())) { LOG.info("Restoring cluster - starting initial backup master: " -+ backup.getHostAndPort()); ++ backup.getAddress()); startMaster(backup.getHostname(), backup.getPort()); } } catch (IOException ex) { @@ -400,7 +393,7 @@ public class DistributedHBaseCluster extends HBaseCluster { for (ServerName sn:toStart) { try { if(!clusterManager.is
[hbase] 06/06: HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 199c994a5c2983d9a8c6185913dc673fdf26e9c3 Author: Duo Zhang AuthorDate: Fri Jan 11 16:22:24 2019 +0800 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection --- .../hadoop/hbase/client/AsyncConnectionImpl.java | 24 +- .../hbase/client/AsyncClusterConnection.java | 17 + .../hbase/client/AsyncClusterConnectionImpl.java | 80 +++ .../AsyncRegionReplicaReplayRetryingCaller.java| 146 .../hbase/client/AsyncRegionServerAdmin.java | 5 +- .../hbase/client/ClusterConnectionFactory.java | 2 +- .../hbase/protobuf/ReplicationProtbufUtil.java | 31 +- .../handler/RegionReplicaFlushHandler.java | 3 +- .../hbase/replication/ReplicationEndpoint.java | 35 +- .../RegionReplicaReplicationEndpoint.java | 783 +++-- .../regionserver/ReplicationSource.java| 2 +- .../hbase/client/TestAsyncTableNoncedRetry.java| 2 +- .../TestRegionReplicaReplicationEndpoint.java | 56 +- ...stRegionReplicaReplicationEndpointNoMaster.java | 99 ++- 14 files changed, 627 insertions(+), 658 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index ce6bfac..f6a2149 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -55,7 +55,6 @@ import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse; @@ -65,7 +64,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ * The implementation of AsyncConnection. */ @InterfaceAudience.Private -class AsyncConnectionImpl implements AsyncClusterConnection { +class AsyncConnectionImpl implements AsyncConnection { private static final Logger LOG = LoggerFactory.getLogger(AsyncConnectionImpl.class); @@ -85,7 +84,7 @@ class AsyncConnectionImpl implements AsyncClusterConnection { private final int rpcTimeout; - private final RpcClient rpcClient; + protected final RpcClient rpcClient; final RpcControllerFactory rpcControllerFactory; @@ -160,16 +159,10 @@ class AsyncConnectionImpl implements AsyncClusterConnection { } // ditto - @Override - public NonceGenerator getNonceGenerator() { + NonceGenerator getNonceGenerator() { return nonceGenerator; } - @Override - public RpcClient getRpcClient() { -return rpcClient; - } - private ClientService.Interface createRegionServerStub(ServerName serverName) throws IOException { return ClientService.newStub(rpcClient.createRpcChannel(serverName, user, rpcTimeout)); } @@ -360,15 +353,4 @@ class AsyncConnectionImpl implements AsyncClusterConnection { return new HBaseHbck(MasterProtos.HbckService.newBlockingStub( rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)), rpcControllerFactory); } - - public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { -return new AsyncRegionServerAdmin(serverName, this); - } - - @Override - public CompletableFuture flush(byte[] regionName, - boolean writeFlushWALMarker) { -RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin(); -return admin.flushRegionInternal(regionName, writeFlushWALMarker); - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java similarity index 72% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index f1f64ca..0ad77ba 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hbase.client; +import java.util.List; import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableN
[hbase] 03/06: HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit e0c580dcc86f220d364589cf9ad9bd034d11193b Author: zhangduo AuthorDate: Thu Dec 6 21:25:34 2018 +0800 HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin --- .../hbase/client/AsyncClusterConnection.java | 6 + .../hadoop/hbase/client/AsyncConnectionImpl.java | 4 + .../hbase/client/AsyncRegionServerAdmin.java | 210 + .../org/apache/hadoop/hbase/util/FutureUtils.java | 2 +- .../org/apache/hadoop/hbase/master/HMaster.java| 15 +- .../apache/hadoop/hbase/master/ServerManager.java | 67 --- .../master/procedure/RSProcedureDispatcher.java| 44 +++-- 7 files changed, 262 insertions(+), 86 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index c7dea25..1327fd7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.yetus.audience.InterfaceAudience; @@ -27,6 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience; public interface AsyncClusterConnection extends AsyncConnection { /** + * Get the admin service for the given region server. + */ + AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName); + + /** * Get the nonce generator for this connection. */ NonceGenerator getNonceGenerator(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 50e27c4..9bead83 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -359,4 +359,8 @@ class AsyncConnectionImpl implements AsyncClusterConnection { return new HBaseHbck(MasterProtos.HbckService.newBlockingStub( rpcClient.createBlockingRpcChannel(masterServer, user, rpcTimeout)), rpcControllerFactory); } + + public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { +return new AsyncRegionServerAdmin(serverName, this); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java new file mode 100644 index 000..9accd89 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ipc.HBaseRpcController; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.ge
[hbase] 05/06: HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit a2453f4e3cace29036c6f1a936926e5def2b11ce Author: Duo Zhang AuthorDate: Wed Dec 12 09:33:33 2018 +0800 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection --- .../hbase/client/AsyncClusterConnection.java | 8 ++ .../hadoop/hbase/client/AsyncConnectionImpl.java | 8 ++ .../hbase/client/ClusterConnectionFactory.java | 16 +-- .../hadoop/hbase/client/RawAsyncHBaseAdmin.java| 36 --- .../org/apache/hadoop/hbase/util/FutureUtils.java | 22 + .../master/procedure/RSProcedureDispatcher.java| 34 +-- .../hbase/protobuf/ReplicationProtbufUtil.java | 15 +-- .../hadoop/hbase/regionserver/HRegionServer.java | 3 +- .../handler/RegionReplicaFlushHandler.java | 110 - 9 files changed, 132 insertions(+), 120 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java index 1327fd7..f1f64ca 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hbase.client; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; + /** * The asynchronous connection for internal usage. */ @@ -41,4 +44,9 @@ public interface AsyncClusterConnection extends AsyncConnection { * Get the rpc client we used to communicate with other servers. */ RpcClient getRpcClient(); + + /** + * Flush a region and get the response. + */ + CompletableFuture flush(byte[] regionName, boolean writeFlushWALMarker); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 9bead83..ce6bfac 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -55,6 +55,7 @@ import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse; @@ -363,4 +364,11 @@ class AsyncConnectionImpl implements AsyncClusterConnection { public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) { return new AsyncRegionServerAdmin(serverName, this); } + + @Override + public CompletableFuture flush(byte[] regionName, + boolean writeFlushWALMarker) { +RawAsyncHBaseAdmin admin = (RawAsyncHBaseAdmin) getAdmin(); +return admin.flushRegionInternal(regionName, writeFlushWALMarker); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java index 68c0630..79484db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java @@ -18,15 +18,12 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; -import java.io.InterruptedIOException; import java.net.SocketAddress; -import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.com.google.common.base.Throwables; - /** * The factory for creating {@link AsyncClusterConnection}. */ @@ -48,16 +45,7 @@ public final class ClusterConnectionFactory { public static AsyncClusterConnection createAsyncClusterConnection(Configuration conf, SocketAddress localAddress, User user) throws IOException { AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf); -String clusterId; -try { - clusterId = registry.getClusterId().get(); -} catch (InterruptedException
[hbase] 01/06: HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 16c0c32b09e742a926645d5be3eaaec63c9c99e6 Author: zhangduo AuthorDate: Fri Nov 30 08:23:47 2018 +0800 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer --- .../hbase/client/AsyncClusterConnection.java | 38 + .../hadoop/hbase/client/AsyncConnectionImpl.java | 14 +++-- .../hbase/client/ClusterConnectionFactory.java | 63 ++ .../hadoop/hbase/client/ConnectionFactory.java | 5 +- .../apache/hadoop/hbase/util/ReflectionUtils.java | 22 .../main/java/org/apache/hadoop/hbase/Server.java | 20 +++ .../org/apache/hadoop/hbase/master/HMaster.java| 3 ++ .../hadoop/hbase/regionserver/HRegionServer.java | 56 +-- .../regionserver/ReplicationSyncUp.java| 6 +++ .../hadoop/hbase/MockRegionServerServices.java | 5 ++ .../client/TestAsyncNonMetaRegionLocator.java | 2 +- ...stAsyncNonMetaRegionLocatorConcurrenyLimit.java | 2 +- .../client/TestAsyncRegionLocatorTimeout.java | 2 +- .../TestAsyncSingleRequestRpcRetryingCaller.java | 4 +- .../hbase/client/TestAsyncTableNoncedRetry.java| 2 +- .../hbase/master/MockNoopMasterServices.java | 6 +++ .../hadoop/hbase/master/MockRegionServer.java | 5 ++ .../hbase/master/TestActiveMasterManager.java | 6 +++ .../hbase/master/cleaner/TestHFileCleaner.java | 6 +++ .../hbase/master/cleaner/TestHFileLinkCleaner.java | 6 +++ .../hbase/master/cleaner/TestLogsCleaner.java | 6 +++ .../cleaner/TestReplicationHFileCleaner.java | 6 +++ .../hbase/regionserver/TestHeapMemoryManager.java | 6 +++ .../hbase/regionserver/TestSplitLogWorker.java | 6 +++ .../hadoop/hbase/regionserver/TestWALLockup.java | 6 +++ .../replication/TestReplicationTrackerZKImpl.java | 6 +++ .../regionserver/TestReplicationSourceManager.java | 6 +++ .../security/token/TestTokenAuthentication.java| 6 +++ .../org/apache/hadoop/hbase/util/MockServer.java | 6 +++ 29 files changed, 290 insertions(+), 37 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java new file mode 100644 index 000..c7dea25 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.ipc.RpcClient; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * The asynchronous connection for internal usage. + */ +@InterfaceAudience.Private +public interface AsyncClusterConnection extends AsyncConnection { + + /** + * Get the nonce generator for this connection. + */ + NonceGenerator getNonceGenerator(); + + /** + * Get the rpc client we used to communicate with other servers. + */ + RpcClient getRpcClient(); +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java index 3cbd950..50e27c4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.client.NonceGenerator.CLIENT_NONCES_ENABLE import static org.apache.hadoop.hbase.util.FutureUtils.addListener; import java.io.IOException; +import java.net.SocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -63,7 +64,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterServ * The implementation of AsyncConnection. */ @InterfaceAudience.Private -class AsyncConnectionImpl implements AsyncConnection { +class AsyncConnectionImpl implements AsyncClusterConnecti
[hbase] 02/06: HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit ec42d7ec75be580996fa5853987d5e85707f9d51 Author: zhangduo AuthorDate: Sat Dec 1 21:15:48 2018 +0800 HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager --- .../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 5 +- .../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 7 ++- .../hadoop/hbase/regionserver/HRegionServer.java | 2 +- .../hbase/regionserver/SecureBulkLoadManager.java | 24 - .../hadoop/hbase/security/token/TokenUtil.java | 57 +- .../hadoop/hbase/security/token/TestTokenUtil.java | 42 6 files changed, 96 insertions(+), 41 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index a3d49b5..d9e620b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -261,13 +261,12 @@ public final class ProtobufUtil { * just {@link ServiceException}. Prefer this method to * {@link #getRemoteException(ServiceException)} because trying to * contain direct protobuf references. - * @param e */ - public static IOException handleRemoteException(Exception e) { + public static IOException handleRemoteException(Throwable e) { return makeIOExceptionOfException(e); } - private static IOException makeIOExceptionOfException(Exception e) { + private static IOException makeIOExceptionOfException(Throwable e) { Throwable t = e; if (e instanceof ServiceException || e instanceof org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index fea81f1..de2fb7d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ByteBufferExtendedCell; @@ -123,6 +122,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.Service; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; + import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest; @@ -343,13 +343,12 @@ public final class ProtobufUtil { * just {@link ServiceException}. Prefer this method to * {@link #getRemoteException(ServiceException)} because trying to * contain direct protobuf references. - * @param e */ - public static IOException handleRemoteException(Exception e) { + public static IOException handleRemoteException(Throwable e) { return makeIOExceptionOfException(e); } - private static IOException makeIOExceptionOfException(Exception e) { + private static IOException makeIOExceptionOfException(Throwable e) { Throwable t = e; if (e instanceof ServiceException) { t = e.getCause(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index b4b1d3e..bbc9d3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1937,7 +1937,7 @@ public class HRegionServer extends HasThread implements if (!isStopped() && !isAborted()) { initializeThreads(); } -this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, clusterConnection); +this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, asyncClusterConnection); this.secureBulkLoadManager.start(); // Health checker thread. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java index 566a6b6..add6519 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBul
[hbase] 04/06: HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 132fe1c1833584bfb09e4a1e18b13e5e7b5a9198 Author: zhangduo AuthorDate: Tue Jan 1 21:27:14 2019 +0800 HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint --- .../hbase/client/AsyncRegionServerAdmin.java | 14 +--- .../hbase/protobuf/ReplicationProtbufUtil.java | 35 ++- .../HBaseInterClusterReplicationEndpoint.java | 31 + .../regionserver/ReplicationSinkManager.java | 40 -- .../hbase/replication/SyncReplicationTestBase.java | 12 +++ .../regionserver/TestReplicationSinkManager.java | 21 +--- 6 files changed, 74 insertions(+), 79 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java index 9accd89..b9141a9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.yetus.audience.InterfaceAudience; @@ -94,9 +95,9 @@ public class AsyncRegionServerAdmin { void call(AdminService.Interface stub, HBaseRpcController controller, RpcCallback done); } - private CompletableFuture call(RpcCall rpcCall) { + private CompletableFuture call(RpcCall rpcCall, CellScanner cellScanner) { CompletableFuture future = new CompletableFuture<>(); -HBaseRpcController controller = conn.rpcControllerFactory.newController(); +HBaseRpcController controller = conn.rpcControllerFactory.newController(cellScanner); try { rpcCall.call(conn.getAdminStub(server), controller, new RpcCallback() { @@ -115,6 +116,10 @@ public class AsyncRegionServerAdmin { return future; } + private CompletableFuture call(RpcCall rpcCall) { +return call(rpcCall, null); + } + public CompletableFuture getRegionInfo(GetRegionInfoRequest request) { return call((stub, controller, done) -> stub.getRegionInfo(controller, request, done)); } @@ -154,8 +159,9 @@ public class AsyncRegionServerAdmin { } public CompletableFuture replicateWALEntry( - ReplicateWALEntryRequest request) { -return call((stub, controller, done) -> stub.replicateWALEntry(controller, request, done)); + ReplicateWALEntryRequest request, CellScanner cellScanner) { +return call((stub, controller, done) -> stub.replicateWALEntry(controller, request, done), + cellScanner); } public CompletableFuture replay(ReplicateWALEntryRequest request) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java index c1b3911..74fad26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java @@ -20,51 +20,54 @@ package org.apache.hadoop.hbase.protobuf; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin; import org.apache.hadoop.hbase.io.SizedCellScanner; -import org.apache.hadoop.hbase.ipc.HBaseRpcController; -import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl; import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.base.Throwables; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; @InterfaceAudience.Private public class ReplicationProtbufUtil { + /** - * A helper to replicate a list of WAL entries using admin protocol. - * @param admin Admin service + * A helper to r
[hbase] branch HBASE-21512 updated (d6fbe51 -> 199c994)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-21512 in repository https://gitbox.apache.org/repos/asf/hbase.git. discard d6fbe51 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection discard 19c9908 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection discard 314094b HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint discard b542ca8 HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin discard ebf6f9c HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager discard f9914c5 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer add 2b5b190 HBASE-21295 Update compatibility matrices add 3d2580c HBASE-21704 The implementation of DistributedHBaseCluster.getServerHoldingRegion is incorrect new 16c0c32 HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer new ec42d7e HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager new e0c580d HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin new 132fe1c HBASE-21579 Use AsyncClusterConnection for HBaseInterClusterReplicationEndpoint new a2453f4 HBASE-21538 Rewrite RegionReplicaFlushHandler to use AsyncClusterConnection new 199c994 HBASE-21671 Rewrite RegionReplicaReplicationEndpoint to use AsyncClusterConnection This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (d6fbe51) \ N -- N -- N refs/heads/HBASE-21512 (199c994) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 6 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/hbase/DistributedHBaseCluster.java | 47 ++ .../java/org/apache/hadoop/hbase/HBaseCluster.java | 5 +- src/main/asciidoc/_chapters/configuration.adoc | 75 -- 3 files changed, 61 insertions(+), 66 deletions(-)
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 8c35057 INFRA-10751 Empty commit 8c35057 is described below commit 8c3505755ac66304539245259b4f7cc484534e29 Author: jenkins AuthorDate: Fri Jan 11 14:48:17 2019 + INFRA-10751 Empty commit
[hbase] 02/04: HBASE-21208 Bytes#toShort doesn't work without unsafe
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 2f541f5710784877787d47f2b0036561e5aff160 Author: Chia-Ping Tsai AuthorDate: Tue Sep 25 10:18:47 2018 +0800 HBASE-21208 Bytes#toShort doesn't work without unsafe Signed-off-by: Ted Yu Signed-off-by: anoopsamjohn Signed-off-by: Reid Chan Conflicts: hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java Amending-Author: Andrew Purtell --- .../java/org/apache/hadoop/hbase/util/Bytes.java | 13 ++- .../org/apache/hadoop/hbase/util/TestBytes.java| 45 ++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index 8e55b63..765f51b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -129,7 +129,8 @@ public class Bytes implements Comparable { // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?) public static final int ESTIMATED_HEAP_TAX = 16; - private static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned(); + @VisibleForTesting + static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned(); /** * Returns length of the byte array, returning 0 if the array is null. @@ -1082,7 +1083,15 @@ public class Bytes implements Comparable { if (length != SIZEOF_SHORT || offset + length > bytes.length) { throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); } -return ConverterHolder.BEST_CONVERTER.toShort(bytes, offset, length); +if (UNSAFE_UNALIGNED) { + return ConverterHolder.BEST_CONVERTER.toShort(bytes, offset, length); +} else { + short n = 0; + n = (short) (n ^ (bytes[offset] & 0xFF)); + n = (short) (n << 8); + n = (short) (n ^ (bytes[offset + 1] & 0xFF)); + return n; +} } /** diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java index 7771f87..2e19ddd 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java @@ -22,6 +22,8 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -39,6 +41,49 @@ import org.junit.experimental.categories.Category; @Category(SmallTests.class) public class TestBytes extends TestCase { + + private static void setUnsafe(boolean value) throws Exception { +Field field = Bytes.class.getDeclaredField("UNSAFE_UNALIGNED"); +field.setAccessible(true); +Field modifiersField = Field.class.getDeclaredField("modifiers"); +modifiersField.setAccessible(true); +int oldModifiers = field.getModifiers(); +modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL); +try { + field.set(null, value); +} finally { + modifiersField.setInt(field, oldModifiers); +} +assertEquals(Bytes.UNSAFE_UNALIGNED, value); + } + + public void testShort() throws Exception { +testShort(false); + } + + public void testShortUnsafe() throws Exception { +testShort(true); + } + + private static void testShort(boolean unsafe) throws Exception { +setUnsafe(unsafe); +try { + for (short n : Arrays.asList( + Short.MIN_VALUE, + (short) -100, + (short) -1, + (short) 0, + (short) 1, + (short) 300, + Short.MAX_VALUE)) { +byte[] bytes = Bytes.toBytes(n); +assertEquals(Bytes.toShort(bytes, 0, bytes.length), n); + } +} finally { + setUnsafe(UnsafeAvailChecker.unaligned()); +} + } + public void testNullHashCode() { byte [] b = null; Exception ee = null;
[hbase] branch branch-1 updated (582c649 -> f3f3798)
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a change to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git. from 582c649 HBASE-21374 Backport HBASE-21342 to branch-1 new 21cfff8 HBASE-20716 Unsafe access cleanup new 2f541f57 HBASE-21208 Bytes#toShort doesn't work without unsafe new 231e6d5 HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to prevent overflow new f3f3798 HBASE-21164 reportForDuty should do backoff rather than retry The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/hadoop/hbase/filter/FuzzyRowFilter.java | 33 +- .../hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 14 +- .../apache/hadoop/hbase/util/ByteBufferUtils.java | 543 ++--- .../java/org/apache/hadoop/hbase/util/Bytes.java | 220 ++--- .../org/apache/hadoop/hbase/util/RetryCounter.java | 10 + .../java/org/apache/hadoop/hbase/util/Sleeper.java | 31 +- .../org/apache/hadoop/hbase/util/UnsafeAccess.java | 267 ++ .../org/apache/hadoop/hbase/util/TestBytes.java| 45 ++ .../hadoop/hbase/io/hfile/HFileBlockIndex.java | 2 +- .../org/apache/hadoop/hbase/master/HMaster.java| 13 +- .../hadoop/hbase/regionserver/HRegionServer.java | 15 +- .../hbase/util/BoundedPriorityBlockingQueue.java | 2 +- .../TestRegionServerReportForDuty.java | 89 13 files changed, 1114 insertions(+), 170 deletions(-)
[hbase] 01/04: HBASE-20716 Unsafe access cleanup
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 21cfff857dd6f241d345bebe74b31901e4b0efa5 Author: Sahil Aggarwal AuthorDate: Tue Oct 9 22:41:36 2018 +0530 HBASE-20716 Unsafe access cleanup Changes the bytes[] conversion done in Bytes and ByteBufferUtils. Instead of doing check unsafe_aligned available everytime, choose the best converter at startup. Conflicts: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java Amending-Author: Andrew Purtell --- .../apache/hadoop/hbase/filter/FuzzyRowFilter.java | 33 +- .../apache/hadoop/hbase/util/ByteBufferUtils.java | 543 ++--- .../java/org/apache/hadoop/hbase/util/Bytes.java | 213 +--- .../org/apache/hadoop/hbase/util/UnsafeAccess.java | 267 ++ 4 files changed, 918 insertions(+), 138 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java index d93d234..a2bcfb8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.UnsafeAccess; import org.apache.hadoop.hbase.util.UnsafeAvailChecker; import com.google.common.annotations.VisibleForTesting; @@ -342,19 +341,13 @@ public class FuzzyRowFilter extends FilterBase { } length = Math.min(length, fuzzyKeyBytes.length); int numWords = length / Bytes.SIZEOF_LONG; -int offsetAdj = offset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; int j = numWords << 3; // numWords * SIZEOF_LONG; for (int i = 0; i < j; i += Bytes.SIZEOF_LONG) { - - long fuzzyBytes = - UnsafeAccess.theUnsafe.getLong(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) i); - long fuzzyMeta = - UnsafeAccess.theUnsafe.getLong(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) i); - long rowValue = UnsafeAccess.theUnsafe.getLong(row, offsetAdj + (long) i); + long fuzzyBytes = Bytes.toLong(fuzzyKeyBytes, i); + long fuzzyMeta = Bytes.toLong(fuzzyKeyMeta, i); + long rowValue = Bytes.toLong(row, offset + i); if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { // We always return NEXT_EXISTS return SatisfiesCode.NEXT_EXISTS; @@ -364,13 +357,9 @@ public class FuzzyRowFilter extends FilterBase { int off = j; if (length - off >= Bytes.SIZEOF_INT) { - int fuzzyBytes = - UnsafeAccess.theUnsafe.getInt(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) off); - int fuzzyMeta = - UnsafeAccess.theUnsafe.getInt(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) off); - int rowValue = UnsafeAccess.theUnsafe.getInt(row, offsetAdj + (long) off); + int fuzzyBytes = Bytes.toInt(fuzzyKeyBytes, off); + int fuzzyMeta = Bytes.toInt(fuzzyKeyMeta, off); + int rowValue = Bytes.toInt(row, offset + off); if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { // We always return NEXT_EXISTS return SatisfiesCode.NEXT_EXISTS; @@ -379,13 +368,9 @@ public class FuzzyRowFilter extends FilterBase { } if (length - off >= Bytes.SIZEOF_SHORT) { - short fuzzyBytes = - UnsafeAccess.theUnsafe.getShort(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) off); - short fuzzyMeta = - UnsafeAccess.theUnsafe.getShort(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET - + (long) off); - short rowValue = UnsafeAccess.theUnsafe.getShort(row, offsetAdj + (long) off); + short fuzzyBytes = Bytes.toShort(fuzzyKeyBytes, off); + short fuzzyMeta = Bytes.toShort(fuzzyKeyMeta, off); + short rowValue = Bytes.toShort(row, offset + off); if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { // We always return NEXT_EXISTS // even if it does not (in this case getNextForFuzzyRule diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java index b5b1d96..36fdec5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.jav
[hbase] 04/04: HBASE-21164 reportForDuty should do backoff rather than retry
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git commit f3f3798575527df45961f4cbdb6d4c1d04cfb1e3 Author: Mingliang Liu AuthorDate: Thu Sep 6 23:01:52 2018 -0700 HBASE-21164 reportForDuty should do backoff rather than retry Remove unused methods from Sleeper (its ok, its @Private). Remove notion of startTime from Sleeper handling (it is is unused). Allow passing in how long to sleep so can maintain externally. In HRS, use a RetryCounter to calculate backoff sleep time for when reportForDuty is failing against a struggling Master. Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java Amending-Author: Andrew Purtell --- .../org/apache/hadoop/hbase/util/RetryCounter.java | 10 +++ .../java/org/apache/hadoop/hbase/util/Sleeper.java | 31 +++- .../org/apache/hadoop/hbase/master/HMaster.java| 13 +++- .../hadoop/hbase/regionserver/HRegionServer.java | 15 +++- .../TestRegionServerReportForDuty.java | 89 ++ 5 files changed, 132 insertions(+), 26 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java index 73512fa..881dd36 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounter.java @@ -174,4 +174,14 @@ public class RetryCounter { public int getAttemptTimes() { return attempts; } + + public long getBackoffTime() { +return this.retryConfig.backoffPolicy.getBackoffTime(this.retryConfig, getAttemptTimes()); + } + + public long getBackoffTimeAndIncrementAttempts() { +long backoffTime = getBackoffTime(); +useRetry(); +return backoffTime; + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java index a60c571..a07ee9c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java @@ -50,13 +50,6 @@ public class Sleeper { } /** - * Sleep for period. - */ - public void sleep() { -sleep(System.currentTimeMillis()); - } - - /** * If currently asleep, stops sleeping; if not asleep, will skip the next * sleep cycle. */ @@ -68,28 +61,24 @@ public class Sleeper { } /** - * Sleep for period adjusted by passed startTime - * @param startTime Time some task started previous to now. Time to sleep - * will be docked current time minus passed startTime. + * Sleep for period. */ - public void sleep(final long startTime) { + public void sleep() { +sleep(this.period); + } + + public void sleep(long sleepTime) { if (this.stopper.isStopped()) { return; } long now = System.currentTimeMillis(); -long waitTime = this.period - (now - startTime); -if (waitTime > this.period) { - LOG.warn("Calculated wait time > " + this.period + -"; setting to this.period: " + System.currentTimeMillis() + ", " + -startTime); - waitTime = this.period; -} -while (waitTime > 0) { +long currentSleepTime = sleepTime; +while (currentSleepTime > 0) { long woke = -1; try { synchronized (sleepLock) { if (triggerWake) break; - sleepLock.wait(waitTime); + sleepLock.wait(currentSleepTime); } woke = System.currentTimeMillis(); long slept = woke - now; @@ -108,7 +97,7 @@ public class Sleeper { } // Recalculate waitTime. woke = (woke == -1)? System.currentTimeMillis(): woke; - waitTime = this.period - (woke - startTime); + currentSleepTime = this.period - (woke - now); } synchronized(sleepLock) { triggerWake = false; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index c2f99f4..d17e297 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -2656,7 +2656,18 @@ public class HMaster extends HRegionServer implements MasterServices, Server { stop("Stopped by " + Thread.currentThread().getName()); } - void checkServiceStarted() throws ServerNotRunningYetException { + @Override + public void stop(String msg) { +if (!isStopped()) { + super.stop(msg); + if (this.activeMasterManager != null)
[hbase] 03/04: HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to prevent overflow
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git commit 231e6d56b569ca17fe620b1d29749dd211e1de02 Author: Saurabh Singh AuthorDate: Fri Jan 11 12:14:18 2019 -0800 HBASE-20928 Rewrite calculation of midpoint in binarySearch functions to prevent overflow HBASE-20928 Rewrite calculation of midpoint - addendum (Xu Cang) Signed-off-by: tedyu Conflicts: hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java Amending-Author: Andrew Purtell --- .../apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 14 +- .../src/main/java/org/apache/hadoop/hbase/util/Bytes.java | 6 +++--- .../org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java | 2 +- .../hadoop/hbase/util/BoundedPriorityBlockingQueue.java| 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java index e72c685..f210fc5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java @@ -154,15 +154,13 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { private int binarySearch(Cell seekCell, boolean seekBefore) { int low = 0; int high = rowNumber - 1; -int mid = (low + high) >>> 1; +int mid = low + ((high - low) >> 1); int comp = 0; SimpleMutableByteRange row = new SimpleMutableByteRange(); while (low <= high) { - mid = (low + high) >>> 1; + mid = low + ((high - low) >> 1); getRow(mid, row); - comp = comparator.compareRows(row.getBytes(), row.getOffset(), - row.getLength(), seekCell.getRowArray(), seekCell.getRowOffset(), - seekCell.getRowLength()); + comp = compareRows(row, seekCell); if (comp < 0) { low = mid + 1; } else if (comp > 0) { @@ -184,6 +182,12 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { } } + private int compareRows(SimpleMutableByteRange row, Cell seekCell) { +return comparator.compareRows(row.getBytes(), row.getOffset(), + row.getLength(), seekCell.getRowArray(), seekCell.getRowOffset(), + seekCell.getRowLength()); + } + private void getRow(int index, SimpleMutableByteRange row) { int offset = Bytes.toIntUnsafe(rowOffsets.array(), rowOffsets.arrayOffset() + (index << 2)); // index * Bytes.SIZEOF_INT diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index 765f51b..22ade4b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -2145,7 +2145,7 @@ public class Bytes implements Comparable { int high = arr.length - 1; while (low <= high) { - int mid = (low+high) >>> 1; + int mid = low + ((high - low) >> 1); // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. int cmp = comparator.compare(key, offset, length, @@ -2182,7 +2182,7 @@ public class Bytes implements Comparable { int high = arr.length - 1; KeyValue.KeyOnlyKeyValue r = new KeyValue.KeyOnlyKeyValue(); while (low <= high) { - int mid = (low+high) >>> 1; + int mid = low + ((high - low) >> 1); // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. r.setKey(arr[mid], 0, arr[mid].length); @@ -2357,7 +2357,7 @@ public class Bytes implements Comparable { int high = toIndex - 1; while (low <= high) { - int mid = (low + high) >>> 1; + int mid = low + ((high - low) >> 1); int midVal = a[mid] & 0xff; if (midVal < unsignedKey) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index da6f3b6..9148815 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -533,7 +533,7 @@ public class HFileBlockIndex { // keys[low - 1] < key < keys[high + 1] while narrowing down the range. KeyValue.KeyOnlyKeyValue nonRootIndexKV = new KeyValue.KeyOnlyKeyValue(); while (low <= high) { -mid = (low + high) >>> 1; +mid = low + ((high - low) >> 1); // Midkey's offset re
[hbase] branch branch-1 updated: HBASE-21325 Force to terminate regionserver when abort hang in somewhere
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-1 by this push: new e7ff91f HBASE-21325 Force to terminate regionserver when abort hang in somewhere e7ff91f is described below commit e7ff91f35ee9fbf9f66ef6e999a8155368d91753 Author: Guanghao Zhang AuthorDate: Fri Oct 19 19:34:04 2018 +0800 HBASE-21325 Force to terminate regionserver when abort hang in somewhere Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java Amending-Author: Andrew Purtell --- .../hadoop/hbase/regionserver/HRegionServer.java | 39 +- .../regionserver/TestRegionServerAbortTimeout.java | 137 + 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 6e5ce80..5f250c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -51,6 +51,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; +import java.util.Timer; +import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; @@ -312,6 +314,11 @@ public class HRegionServer extends HasThread implements // Go down hard. Used if file system becomes unavailable and also in // debugging and unit tests. private volatile boolean abortRequested; + public static final String ABORT_TIMEOUT = "hbase.regionserver.abort.timeout"; + // Default abort timeout is 1200 seconds for safe + private static final long DEFAULT_ABORT_TIMEOUT = 120; + // Will run this task when abort timeout + public static final String ABORT_TIMEOUT_TASK = "hbase.regionserver.abort.timeout.task"; ConcurrentMap rowlocks = new ConcurrentHashMap(); @@ -1041,12 +1048,31 @@ public class HRegionServer extends HasThread implements abort(prefix + t.getMessage(), t); } } + // Run shutdown. if (mxBean != null) { MBeanUtil.unregisterMBean(mxBean); mxBean = null; } -if (this.leases != null) this.leases.closeAfterLeasesExpire(); + +if (abortRequested) { + Timer abortMonitor = new Timer("Abort regionserver monitor", true); + TimerTask abortTimeoutTask = null; + try { +abortTimeoutTask = +Class.forName(conf.get(ABORT_TIMEOUT_TASK, SystemExitWhenAbortTimeout.class.getName())) + .asSubclass(TimerTask.class).getDeclaredConstructor().newInstance(); + } catch (Exception e) { +LOG.warn("Initialize abort timeout task failed", e); + } + if (abortTimeoutTask != null) { +abortMonitor.schedule(abortTimeoutTask, conf.getLong(ABORT_TIMEOUT, DEFAULT_ABORT_TIMEOUT)); + } +} + +if (this.leases != null) { + this.leases.closeAfterLeasesExpire(); +} if (this.splitLogWorker != null) { splitLogWorker.stop(); } @@ -3552,4 +3578,15 @@ public class HRegionServer extends HasThread implements public void unassign(byte[] regionName) throws IOException { clusterConnection.getAdmin().unassign(regionName, false); } + + /** + * Force to terminate region server when abort timeout. + */ + private static class SystemExitWhenAbortTimeout extends TimerTask { +@Override +public void run() { + LOG.warn("Aborting region server timed out, terminate forcibly..."); + System.exit(1); +} + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java new file mode 100644 index 000..ed129c5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAbortTimeout.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANT
[hbase] branch branch-1 updated: HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region server level) to branch-1
This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-1 by this push: new d6a3e6b HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region server level) to branch-1 d6a3e6b is described below commit d6a3e6b404a775edaf28cefb326e5186fd76b9dd Author: Andrew Purtell AuthorDate: Thu Jan 10 18:07:36 2019 -0800 HBASE-21679 Port HBASE-6028 (Start/Stop compactions at region server level) to branch-1 HBASE-6028 Start/Stop compactions at region server level Add switching on/off of compactions. Switching off compactions will also interrupt any currently ongoing compactions. Adds a "compaction_switch" to hbase shell. Switching off compactions will interrupt any currently ongoing compactions. State set from shell will be lost on restart. To persist the changes across region servers modify hbase.regionserver.compaction.enabled in hbase-site.xml and restart. --- .../java/org/apache/hadoop/hbase/client/Admin.java | 13 + .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 47 + hbase-common/src/main/resources/hbase-default.xml |7 + .../hbase/protobuf/generated/AdminProtos.java | 1268 +--- hbase-protocol/src/main/protobuf/Admin.proto | 11 + .../hbase/regionserver/CompactSplitThread.java | 149 ++- .../hbase/regionserver/CompactionRequestor.java|5 + .../apache/hadoop/hbase/regionserver/HRegion.java | 23 +- .../hadoop/hbase/regionserver/RSRpcServices.java | 22 + .../apache/hadoop/hbase/regionserver/Region.java |2 +- .../hadoop/hbase/master/MockRegionServer.java |8 + .../hadoop/hbase/regionserver/TestCompaction.java | 86 +- .../replication/regionserver/TestReplicator.java |6 + hbase-shell/src/main/ruby/hbase/admin.rb | 17 +- hbase-shell/src/main/ruby/shell.rb |1 + .../main/ruby/shell/commands/compaction_switch.rb | 53 + src/main/asciidoc/_chapters/architecture.adoc |8 + 17 files changed, 1517 insertions(+), 209 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 8f5e150..ea87457 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -634,6 +634,19 @@ public interface Admin extends Abortable, Closeable { throws IOException; /** + * Turn the compaction on or off. Disabling compactions will also interrupt any currently ongoing + * compactions. It is ephemeral. This setting will be lost on restart of the server. Compaction + * can also be enabled/disabled by modifying configuration hbase.regionserver.compaction.enabled + * in hbase-site.xml. + * + * @param switchState Set to true to enable, false to disable. + * @param serverNamesList list of region servers. + * @return Previous compaction states for region servers + */ + Map compactionSwitch(boolean switchState, List serverNamesList) + throws IOException; + + /** * Compact all regions on the region server * @param sn the region server name * @param major if it's major compaction diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index fafc9fe..7e624cc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -83,6 +83,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactionSwitchRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactionSwitchResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse; @@ -2052,6 +2054,51 @@ public class HBaseAdmin implements Admin { * {@inheritDoc} */ @Override + public Map compactionSwitch(boolean switchState, List + serverNamesList) throws IOException { +List serverList = new ArrayList<>(); +if (serverNamesList.isEmpty()) { + ClusterStatus status = getClusterStatus(); + serverList.addAll(status.getServers()); +} else { + for (String regionServerName :