[hbase] branch master updated: HBASE-27528 log duplication issues in MasterRpcServices. (#4951)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 913cf6b96df HBASE-27528 log duplication issues in MasterRpcServices.
(#4951)
913cf6b96df is described below
commit 913cf6b96df1fe28c84be0e6c382090cbcd9d8b7
Author: curie71 <39853223+curi...@users.noreply.github.com>
AuthorDate: Fri Jan 20 15:53:32 2023 +0800
HBASE-27528 log duplication issues in MasterRpcServices. (#4951)
Signed-off-by: Duo Zhang
---
.../org/apache/hadoop/hbase/master/MasterRpcServices.java | 14 --
1 file changed, 14 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index db02d7699bb..f490e8377d1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -2809,13 +2809,6 @@ public class MasterRpcServices extends
HBaseRpcServicesBase
mergeExistingPermissions);
}
server.cpHost.postGrant(perm, mergeExistingPermissions);
-User caller = RpcServer.getRequestUser().orElse(null);
-if (AUDITLOG.isTraceEnabled()) {
- // audit log should store permission changes in addition to auth
results
- String remoteAddress =
RpcServer.getRemoteAddress().map(InetAddress::toString).orElse("");
- AUDITLOG.trace("User {} (remote address: {}) granted permission {}",
caller,
-remoteAddress, perm);
-}
return GrantResponse.getDefaultInstance();
} else {
throw new DoNotRetryIOException(
@@ -2839,13 +2832,6 @@ public class MasterRpcServices extends
HBaseRpcServicesBase
PermissionStorage.removeUserPermission(server.getConfiguration(),
userPermission, table);
}
server.cpHost.postRevoke(userPermission);
-User caller = RpcServer.getRequestUser().orElse(null);
-if (AUDITLOG.isTraceEnabled()) {
- // audit log should record all permission changes
- String remoteAddress =
RpcServer.getRemoteAddress().map(InetAddress::toString).orElse("");
- AUDITLOG.trace("User {} (remote address: {}) revoked permission {}",
caller,
-remoteAddress, userPermission);
-}
return RevokeResponse.getDefaultInstance();
} else {
throw new DoNotRetryIOException(
[hbase] branch master updated: HBASE-27562 Publish SBOM artifacts (#4953)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/master by this push: new 5feb06bd69f HBASE-27562 Publish SBOM artifacts (#4953) 5feb06bd69f is described below commit 5feb06bd69fa23eab4bd778275a034a5d47b446f Author: Dongjoon Hyun AuthorDate: Thu Jan 19 23:53:06 2023 -0800 HBASE-27562 Publish SBOM artifacts (#4953) Signed-off-by: Duo Zhang --- pom.xml | 13 + 1 file changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index b1f78369cbe..1d32c71df9a 100644 --- a/pom.xml +++ b/pom.xml @@ -2866,6 +2866,19 @@ + +org.cyclonedx +cyclonedx-maven-plugin +2.7.3 + + + + makeBom + +package + + +
[hbase] branch branch-2 updated: HBASE-27564 Add default encryption type for MiniKDC to fix failed tests on JDK11+ (#4968)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 8e11f91b3b7 HBASE-27564 Add default encryption type for MiniKDC to fix
failed tests on JDK11+ (#4968)
8e11f91b3b7 is described below
commit 8e11f91b3b7786da518969070a1befeacb783ecc
Author: tianhang
AuthorDate: Fri Jan 20 15:52:39 2023 +0800
HBASE-27564 Add default encryption type for MiniKDC to fix failed tests on
JDK11+ (#4968)
Co-authored-by: tianhang.tang
Signed-off-by: Duo Zhang
---
.../src/test/resources/minikdc-krb5.conf | 27 ++
.../src/test/resources/minikdc-krb5.conf | 27 ++
hbase-http/src/test/resources/minikdc-krb5.conf| 27 ++
.../src/test/resources/minikdc-krb5.conf | 27 ++
4 files changed, 108 insertions(+)
diff --git a/hbase-endpoint/src/test/resources/minikdc-krb5.conf
b/hbase-endpoint/src/test/resources/minikdc-krb5.conf
new file mode 100644
index 000..0987ef2bcc6
--- /dev/null
+++ b/hbase-endpoint/src/test/resources/minikdc-krb5.conf
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[libdefaults]
+default_realm = {0}
+udp_preference_limit = 1
+default_tkt_enctypes = aes128-cts-hmac-sha1-96
+default_tgs_enctypes = aes128-cts-hmac-sha1-96
+
+[realms]
+{0} = '{'
+kdc = {1}:{2}
+'}'
diff --git a/hbase-examples/src/test/resources/minikdc-krb5.conf
b/hbase-examples/src/test/resources/minikdc-krb5.conf
new file mode 100644
index 000..0987ef2bcc6
--- /dev/null
+++ b/hbase-examples/src/test/resources/minikdc-krb5.conf
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[libdefaults]
+default_realm = {0}
+udp_preference_limit = 1
+default_tkt_enctypes = aes128-cts-hmac-sha1-96
+default_tgs_enctypes = aes128-cts-hmac-sha1-96
+
+[realms]
+{0} = '{'
+kdc = {1}:{2}
+'}'
diff --git a/hbase-http/src/test/resources/minikdc-krb5.conf
b/hbase-http/src/test/resources/minikdc-krb5.conf
new file mode 100644
index 000..0987ef2bcc6
--- /dev/null
+++ b/hbase-http/src/test/resources/minikdc-krb5.conf
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[libdefaults]
+default_realm = {0}
+udp_preference_limit = 1
+default_tkt_enctypes = aes128-cts-hmac-sha1-96
+default_tgs_enctypes = aes128-cts-hmac-sha1-96
+
+[realms]
+{0} = '{'
+kdc = {1}:{2}
+'}'
diff --git a/hbase-mapreduce/src/test/resources/minikdc-krb5.conf
[hbase] branch branch-2.4 updated: HBASE-27557 [JDK17] Update shade plugin version (#4970)
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new e63331e93ee HBASE-27557 [JDK17] Update shade plugin version (#4970) e63331e93ee is described below commit e63331e93ee9e5ec42efe185f5ab19fe3a8793e2 Author: tianhang AuthorDate: Fri Jan 20 13:32:47 2023 +0800 HBASE-27557 [JDK17] Update shade plugin version (#4970) Signed-off-by Reid Chan --- hbase-protocol-shaded/pom.xml | 1 + hbase-shaded/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index 2ea3c33f294..dadf086eddb 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -150,6 +150,7 @@ org.apache.maven.plugins maven-shade-plugin +3.4.1 diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index c9d345862e4..0a0f0a1f863 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -78,7 +78,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.4.1 aggregate-into-a-jar-with-relocated-third-parties
[hbase] branch branch-2.5 updated: HBASE-27557 [JDK17] Update shade plugin version (#4984)
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.5 by this push: new 5eb81363acf HBASE-27557 [JDK17] Update shade plugin version (#4984) 5eb81363acf is described below commit 5eb81363acf03f8537444a7cd84542e02c8757f9 Author: tianhang AuthorDate: Fri Jan 20 10:52:36 2023 +0800 HBASE-27557 [JDK17] Update shade plugin version (#4984) Signed-off-by: Reid Chan --- hbase-protocol-shaded/pom.xml | 1 + hbase-shaded/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index bd2c48cea5a..f478b28fa40 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -146,6 +146,7 @@ org.apache.maven.plugins maven-shade-plugin +3.4.1 diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 10b410cfd33..100595de436 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -83,7 +83,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.4.1 aggregate-into-a-jar-with-relocated-third-parties
[hbase] branch branch-2 updated: HBASE-27557 [JDK17] Update shade plugin version (#4983)
This is an automated email from the ASF dual-hosted git repository. reidchan pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 750e38f7703 HBASE-27557 [JDK17] Update shade plugin version (#4983) 750e38f7703 is described below commit 750e38f770379a6aa067e37d363130926a499a35 Author: tianhang AuthorDate: Fri Jan 20 10:52:27 2023 +0800 HBASE-27557 [JDK17] Update shade plugin version (#4983) Signed-off-by: Reid Chan --- hbase-protocol-shaded/pom.xml | 1 + hbase-shaded/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index bd2c48cea5a..f478b28fa40 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -146,6 +146,7 @@ org.apache.maven.plugins maven-shade-plugin +3.4.1 diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml index 10b410cfd33..100595de436 100644 --- a/hbase-shaded/pom.xml +++ b/hbase-shaded/pom.xml @@ -83,7 +83,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.4.1 aggregate-into-a-jar-with-relocated-third-parties
[hbase] branch branch-2.4 updated: HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980)
This is an automated email from the ASF dual-hosted git repository. taklwu pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 9e2a17c4afc HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) 9e2a17c4afc is described below commit 9e2a17c4afc07a5fffbd6da208d85dd7c21b8cc9 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Thu Jan 19 18:03:33 2023 -0800 HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) Bumps [future](https://github.com/PythonCharmers/python-future) from 0.18.2 to 0.18.3. - [Release notes](https://github.com/PythonCharmers/python-future/releases) - [Changelog](https://github.com/PythonCharmers/python-future/blob/master/docs/changelog.rst) - [Commits](https://github.com/PythonCharmers/python-future/compare/v0.18.2...v0.18.3) --- updated-dependencies: - dependency-name: future dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Andrew Purtell Signed-off-by: Duo Zhang Signed-off-by: Tak Lon (Stephen) Wu --- dev-support/python-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/python-requirements.txt b/dev-support/python-requirements.txt index 3d596db45c8..2377064eaa1 100644 --- a/dev-support/python-requirements.txt +++ b/dev-support/python-requirements.txt @@ -16,7 +16,7 @@ # limitations under the License. # requests==2.28.1 -future==0.18.2 +future==0.18.3 gitpython==3.1.30 rbtools==4.0 jinja2==3.1.2
[hbase] branch branch-2.5 updated: HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980)
This is an automated email from the ASF dual-hosted git repository. taklwu pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2.5 by this push: new 590b6fb5886 HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) 590b6fb5886 is described below commit 590b6fb5886f6e61cda687f1c5b77c52bafd8dba Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Thu Jan 19 18:03:33 2023 -0800 HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) Bumps [future](https://github.com/PythonCharmers/python-future) from 0.18.2 to 0.18.3. - [Release notes](https://github.com/PythonCharmers/python-future/releases) - [Changelog](https://github.com/PythonCharmers/python-future/blob/master/docs/changelog.rst) - [Commits](https://github.com/PythonCharmers/python-future/compare/v0.18.2...v0.18.3) --- updated-dependencies: - dependency-name: future dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Andrew Purtell Signed-off-by: Duo Zhang Signed-off-by: Tak Lon (Stephen) Wu --- dev-support/python-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/python-requirements.txt b/dev-support/python-requirements.txt index 3d596db45c8..2377064eaa1 100644 --- a/dev-support/python-requirements.txt +++ b/dev-support/python-requirements.txt @@ -16,7 +16,7 @@ # limitations under the License. # requests==2.28.1 -future==0.18.2 +future==0.18.3 gitpython==3.1.30 rbtools==4.0 jinja2==3.1.2
[hbase] branch branch-2 updated: HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980)
This is an automated email from the ASF dual-hosted git repository. taklwu pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new 72aaa0b4069 HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) 72aaa0b4069 is described below commit 72aaa0b4069423f2ff09cdd65efb57c1ed2f51b2 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Thu Jan 19 18:03:33 2023 -0800 HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) Bumps [future](https://github.com/PythonCharmers/python-future) from 0.18.2 to 0.18.3. - [Release notes](https://github.com/PythonCharmers/python-future/releases) - [Changelog](https://github.com/PythonCharmers/python-future/blob/master/docs/changelog.rst) - [Commits](https://github.com/PythonCharmers/python-future/compare/v0.18.2...v0.18.3) --- updated-dependencies: - dependency-name: future dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Andrew Purtell Signed-off-by: Duo Zhang Signed-off-by: Tak Lon (Stephen) Wu --- dev-support/python-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/python-requirements.txt b/dev-support/python-requirements.txt index 3d596db45c8..2377064eaa1 100644 --- a/dev-support/python-requirements.txt +++ b/dev-support/python-requirements.txt @@ -16,7 +16,7 @@ # limitations under the License. # requests==2.28.1 -future==0.18.2 +future==0.18.3 gitpython==3.1.30 rbtools==4.0 jinja2==3.1.2
[hbase] branch master updated (ad8f28e2978 -> 29d4632b13f)
This is an automated email from the ASF dual-hosted git repository. taklwu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git from ad8f28e2978 HBASE-27493: Allow namespace admins to clone snapshots created by them (#4885) add 29d4632b13f HBASE-27575 Bump future from 0.18.2 to 0.18.3 in /dev-support (#4980) No new revisions were added by this update. Summary of changes: dev-support/python-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[hbase] 09/10: HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f852b3f8cb25439cfe6330ff65f0764ef3b42c8c
Author: Duo Zhang
AuthorDate: Tue Oct 18 16:46:03 2022 +0800
HBASE-27429 Add exponential retry backoff support for
MigrateReplicationQueueFromZkToTableProcedure
Signed-off-by: Liangjun He
---
.../hbase/procedure2/TimeoutExecutorThread.java| 10 +-
...rateReplicationQueueFromZkToTableProcedure.java | 131 ++---
.../master/replication/ReplicationPeerManager.java | 45 ---
...tReplicationPeerManagerMigrateQueuesFromZk.java | 9 +-
4 files changed, 125 insertions(+), 70 deletions(-)
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
index 3b99781a558..c0287a99435 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
@@ -78,9 +78,13 @@ class TimeoutExecutorThread extends
StoppableThread {
}
public void add(Procedure procedure) {
-LOG.info("ADDED {}; timeout={}, timestamp={}", procedure,
procedure.getTimeout(),
- procedure.getTimeoutTimestamp());
-queue.add(new DelayedProcedure<>(procedure));
+if (procedure.getTimeout() > 0) {
+ LOG.info("ADDED {}; timeout={}, timestamp={}", procedure,
procedure.getTimeout(),
+procedure.getTimeoutTimestamp());
+ queue.add(new DelayedProcedure<>(procedure));
+} else {
+ LOG.info("Got negative timeout {} for {}, skip adding",
procedure.getTimeout(), procedure);
+}
}
public boolean remove(Procedure procedure) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
index 536f232338e..93ff27db3f7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
@@ -25,19 +25,25 @@ import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureP
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.function.LongConsumer;
import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.master.procedure.GlobalProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import
org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.util.FutureUtils;
+import org.apache.hadoop.hbase.util.IdLock;
+import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
@@ -65,18 +71,34 @@ public class MigrateReplicationQueueFromZkToTableProcedure
private List disabledPeerIds;
- private List> futures;
+ private CompletableFuture future;
private ExecutorService executor;
+ private RetryCounter retryCounter;
+
@Override
public String getGlobalId() {
return getClass().getSimpleName();
}
+ private ProcedureSuspendedException suspend(Configuration conf, LongConsumer
backoffConsumer)
+throws ProcedureSuspendedException {
+if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(conf);
+}
+long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+backoffConsumer.accept(backoff);
+throw suspend(Math.toIntExact(backoff), true);
+ }
+
+ private void resetRetry() {
+retryCounter = null;
+ }
+
private ExecutorService getExecutorService() {
if (executor == null) {
- executor = Executors.newFixedThreadPool(3, new ThreadFactoryBuilder()
+ executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder()
[hbase] 05/10: HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fb842ac21d73d2739df312206548c6a2881e3853
Author: Duo Zhang
AuthorDate: Thu Sep 29 10:08:02 2022 +0800
HBASE-27392 Add a new procedure type for implementing some global
operations such as migration (#4803)
Signed-off-by: Xin Sun
---
.../hbase/procedure2/LockedResourceType.java | 3 +-
.../master/procedure/GlobalProcedureInterface.java | 15 ++-
.../hadoop/hbase/master/procedure/GlobalQueue.java | 21 ++--
.../master/procedure/MasterProcedureScheduler.java | 119 -
.../hbase/master/procedure/SchemaLocking.java | 18 +++-
.../procedure/TestMasterProcedureScheduler.java| 48 +
6 files changed, 202 insertions(+), 22 deletions(-)
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index 12f899d7565..40141017009 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -26,5 +26,6 @@ public enum LockedResourceType {
TABLE,
REGION,
PEER,
- META
+ META,
+ GLOBAL
}
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
similarity index 82%
copy from
hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
copy to
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
index 12f899d7565..1ef168abfd8 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.procedure2;
+package org.apache.hadoop.hbase.master.procedure;
import org.apache.yetus.audience.InterfaceAudience;
+/**
+ * Procedure interface for global operations, such as migration.
+ */
@InterfaceAudience.Private
-public enum LockedResourceType {
- SERVER,
- NAMESPACE,
- TABLE,
- REGION,
- PEER,
- META
+public interface GlobalProcedureInterface {
+
+ String getGlobalId();
}
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
similarity index 69%
copy from
hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
copy to
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
index 12f899d7565..1633dc4856e 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
@@ -15,16 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.procedure2;
+package org.apache.hadoop.hbase.master.procedure;
+import org.apache.hadoop.hbase.procedure2.LockStatus;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
-public enum LockedResourceType {
- SERVER,
- NAMESPACE,
- TABLE,
- REGION,
- PEER,
- META
+public class GlobalQueue extends Queue {
+
+ public GlobalQueue(String globalId, LockStatus lockStatus) {
+super(globalId, lockStatus);
+ }
+
+ @Override
+ boolean requireExclusiveLock(Procedure proc) {
+return true;
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 866f2f6f403..fbf0eb8abf3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@@ -95,16 +96,20 @@ public class MasterProcedureScheduler extends
AbstractProcedureScheduler {
(n, k) -> n.compareKey((String) k);
private
[hbase] 07/10: HBASE-27218 Support rolling upgrading (#4808)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit da08965d7c78c71a4af97f393fb6cd60f3feb9d3
Author: Duo Zhang
AuthorDate: Sun Nov 6 16:57:11 2022 +0800
HBASE-27218 Support rolling upgrading (#4808)
Signed-off-by: Yu Li
---
.../apache/hadoop/hbase/zookeeper/ZNodePaths.java | 8 +-
.../apache/hadoop/hbase/procedure2/Procedure.java | 15 +
.../protobuf/server/master/MasterProcedure.proto | 12 +
hbase-replication/pom.xml | 10 +
.../hbase/replication/ReplicationQueueStorage.java | 19 ++
.../replication/TableReplicationQueueStorage.java | 65 +++-
.../ZKReplicationQueueStorageForMigration.java | 351 +
.../replication/TestZKReplicationQueueStorage.java | 317 +++
hbase-server/pom.xml | 6 +
.../org/apache/hadoop/hbase/master/HMaster.java| 13 +
.../master/procedure/ServerCrashProcedure.java | 19 ++
.../replication/AbstractPeerNoLockProcedure.java | 5 +-
...rateReplicationQueueFromZkToTableProcedure.java | 244 ++
.../master/replication/ModifyPeerProcedure.java| 26 ++
.../master/replication/ReplicationPeerManager.java | 104 +-
.../TransitPeerSyncReplicationStateProcedure.java | 14 +
.../replication/TestMigrateReplicationQueue.java | 126
...rateReplicationQueueFromZkToTableProcedure.java | 226 +
...icationQueueFromZkToTableProcedureRecovery.java | 128
...tReplicationPeerManagerMigrateQueuesFromZk.java | 216 +
.../hbase/replication/TestReplicationBase.java | 2 +-
pom.xml| 7 +-
22 files changed, 1917 insertions(+), 16 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
index d19d2100466..3f66c7cdc0c 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
@@ -220,7 +220,11 @@ public class ZNodePaths {
* @param suffix ending of znode name
* @return result of properly joining prefix with suffix
*/
- public static String joinZNode(String prefix, String suffix) {
-return prefix + ZNodePaths.ZNODE_PATH_SEPARATOR + suffix;
+ public static String joinZNode(String prefix, String... suffix) {
+StringBuilder sb = new StringBuilder(prefix);
+for (String s : suffix) {
+ sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(s);
+}
+return sb.toString();
}
}
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 34c74d92c16..43adba2bc21 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.metrics.Counter;
import org.apache.hadoop.hbase.metrics.Histogram;
@@ -33,6 +34,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
/**
@@ -1011,6 +1013,19 @@ public abstract class Procedure implements
Comparable other) {
return Long.compare(getProcId(), other.getProcId());
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 76a1d676487..b6f5d7e50bb 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -722,3 +722,15 @@ enum AssignReplicationQueuesState {
message AssignReplicationQueuesStateData {
required ServerName crashed_server = 1;
}
+
+enum MigrateReplicationQueueFromZkToTableState {
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 1;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 2;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 3;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 4;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 5;
+}
+
+message MigrateReplicationQueueFromZkToTableStateData {
+ repeated string disabled_peer_id = 1;
+}
diff --git a/hbase-replication/pom.xml
[hbase] branch HBASE-27109/table_based_rqs updated (ff31d25b3d0 -> bf1f0082824)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch HBASE-27109/table_based_rqs in repository https://gitbox.apache.org/repos/asf/hbase.git discard ff31d25b3d0 HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901) discard ac878a550d2 HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure discard 643e5fc758c HBASE-27217 Revisit the DumpReplicationQueues tool (#4810) discard a258accdade HBASE-27218 Support rolling upgrading (#4808) discard a4218b55620 HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811) discard 93b20a6cd45 HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803) discard 45589dd4fec HBASE-27215 Add support for sync replication (#4762) discard 2bdd5719e70 HBASE-27214 Implement the new replication hfile/log cleaner (#4722) discard bc5eb2269ae HBASE-27213 Add support for claim queue operation (#4708) discard 3cf605d1392 HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672) add ad8f28e2978 HBASE-27493: Allow namespace admins to clone snapshots created by them (#4885) new 5210fa1d089 HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672) new 4a75db14b7a HBASE-27213 Add support for claim queue operation (#4708) new 42a9e92fdc9 HBASE-27214 Implement the new replication hfile/log cleaner (#4722) new 4f5db83ac61 HBASE-27215 Add support for sync replication (#4762) new fb842ac21d7 HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803) new f0332a62498 HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811) new da08965d7c7 HBASE-27218 Support rolling upgrading (#4808) new d2c82c7750d HBASE-27217 Revisit the DumpReplicationQueues tool (#4810) new f852b3f8cb2 HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure new bf1f0082824 HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ff31d25b3d0) \ N -- N -- N refs/heads/HBASE-27109/table_based_rqs (bf1f0082824) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 10 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hbase/security/access/AccessController.java| 3 ++ .../security/access/TestAccessController.java | 63 +++--- 2 files changed, 58 insertions(+), 8 deletions(-)
[hbase] 10/10: HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit bf1f0082824cf3f0aa38abf8c2e20bbd03c60473
Author: Duo Zhang
AuthorDate: Sat Dec 3 20:51:40 2022 +0800
HBASE-27430 Should disable replication log cleaner when migrating
replication queue data (#4901)
Signed-off-by: Liangjun He
---
.../protobuf/server/master/MasterProcedure.proto | 12 +++---
...rateReplicationQueueFromZkToTableProcedure.java | 47 +-
...rateReplicationQueueFromZkToTableProcedure.java | 29 -
3 files changed, 80 insertions(+), 8 deletions(-)
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index b6f5d7e50bb..14d07c17c88 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -724,11 +724,13 @@ message AssignReplicationQueuesStateData {
}
enum MigrateReplicationQueueFromZkToTableState {
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 1;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 2;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 3;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 4;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 5;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER = 1;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 2;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 3;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 4;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 5;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 6;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER = 7;
}
message MigrateReplicationQueueFromZkToTableStateData {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
index 93ff27db3f7..b7c4e33ef85 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER;
import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER;
+import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER;
import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER;
import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE;
import static
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE;
@@ -111,6 +113,26 @@ public class MigrateReplicationQueueFromZkToTableProcedure
}
}
+ private void disableReplicationLogCleaner(MasterProcedureEnv env)
+throws ProcedureSuspendedException {
+if
(!env.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable()) {
+ // it is not likely that we can reach here as we will schedule this
procedure immediately
+ // after master restarting, where ReplicationLogCleaner should have not
started its first run
+ // yet. But anyway, let's make the code more robust. And it is safe to
wait a bit here since
+ // there will be no data in the new replication queue storage before we
execute this procedure
+ // so ReplicationLogCleaner will quit immediately without doing anything.
+ throw suspend(env.getMasterConfiguration(),
+backoff -> LOG.info(
+ "Can not disable replication log cleaner, sleep {} secs and retry
later",
+ backoff / 1000));
+}
+resetRetry();
+ }
+
+ private void enableReplicationLogCleaner(MasterProcedureEnv env) {
+env.getReplicationPeerManager().getReplicationLogCleanerBarrier().enable();
+ }
+
private void waitUntilNoPeerProcedure(MasterProcedureEnv
[hbase] 03/10: HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 42a9e92fdc9c47fa5dee90cd0476914cc0713952
Author: Duo Zhang
AuthorDate: Wed Aug 31 21:24:09 2022 +0800
HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
Signed-off-by: Xin Sun
---
.../org/apache/hadoop/hbase/master/HMaster.java| 1 -
.../hbase/master/cleaner/FileCleanerDelegate.java | 2 +-
.../hadoop/hbase/master/region/MasterRegion.java | 2 +-
.../hbase/master/replication/AddPeerProcedure.java | 15 +-
.../master/replication/ReplicationPeerManager.java | 8 +
.../hadoop/hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/replication/ReplicationOffsetUtil.java | 47 +++
.../replication/master/ReplicationLogCleaner.java | 234 +
.../master/ReplicationLogCleanerBarrier.java | 85 +
.../regionserver/ReplicationSourceManager.java | 18 +-
.../regionserver/ReplicationSyncUp.java| 5 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java| 29 ++
.../org/apache/hadoop/hbase/wal/WALFactory.java| 29 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 227 +---
.../cleaner/TestReplicationHFileCleaner.java | 43 ++-
.../replication/TestReplicationOffsetUtil.java | 52 +++
.../replication/master/TestLogCleanerBarrier.java | 60
.../master/TestReplicationLogCleaner.java | 385 +
.../regionserver/TestReplicationSourceManager.java | 2 +-
.../apache/hadoop/hbase/wal/TestWALFactory.java| 2 +-
.../apache/hadoop/hbase/wal/TestWALMethods.java| 14 +
21 files changed, 1008 insertions(+), 254 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d351fc67d6e..92b26ef36a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4277,5 +4277,4 @@ public class HMaster extends
HBaseServerBase implements Maste
// initialize master side coprocessors before we start handling requests
this.cpHost = new MasterCoprocessorHost(this, conf);
}
-
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
index d37bb620273..e08f5329433 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -50,7 +50,7 @@ public interface FileCleanerDelegate extends Configurable,
Stoppable {
}
/**
- * Used to do some cleanup work
+ * Will be called after cleaner run.
*/
default void postClean() {
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
index 177e161c32e..45f049723c7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
@@ -377,7 +377,7 @@ public final class MasterRegion {
params.archivedWalSuffix(), params.rollPeriodMs(), params.flushSize());
walRoller.start();
-WALFactory walFactory = new WALFactory(conf,
server.getServerName().toString(), server, false);
+WALFactory walFactory = new WALFactory(conf, server.getServerName(),
server, false);
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index 6d0acee76ca..25a4cd4b08e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -84,15 +83,21 @@ public
[hbase] 06/10: HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f0332a62498a38add66403e5dd8ca4a894cb7d59
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Wed Oct 12 14:40:05 2022 +0800
HBASE-27405 Fix the replication hfile/log cleaner report that the
replication table does not exist (#4811)
Signed-off-by: Duo Zhang
---
.../apache/hadoop/hbase/replication/ReplicationQueueStorage.java | 6 ++
.../hadoop/hbase/replication/TableReplicationQueueStorage.java | 9 +
.../hadoop/hbase/replication/master/ReplicationLogCleaner.java | 8
.../hbase/replication/master/TestReplicationLogCleaner.java | 1 +
4 files changed, 24 insertions(+)
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index c4204f0e8c4..6f6aee38cc8 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -178,4 +178,10 @@ public interface ReplicationQueueStorage {
* created hfile references during the call may not be included.
*/
Set getAllHFileRefs() throws ReplicationException;
+
+ /**
+ * Whether the replication queue table exists.
+ * @return Whether the replication queue table exists
+ */
+ boolean hasData() throws ReplicationException;
}
diff --git
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
index 0c9553f4fd8..392a3692d66 100644
---
a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
+++
b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -532,4 +532,13 @@ public class TableReplicationQueueStorage implements
ReplicationQueueStorage {
throw new ReplicationException("failed to getAllHFileRefs", e);
}
}
+
+ @Override
+ public boolean hasData() throws ReplicationException {
+try {
+ return conn.getAdmin().getDescriptor(tableName) != null;
+} catch (IOException e) {
+ throw new ReplicationException("failed to get replication queue table",
e);
+}
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index f1fd8f8d6b3..3ab52da6158 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -76,6 +76,14 @@ public class ReplicationLogCleaner extends
BaseLogCleanerDelegate {
if (this.getConf() == null) {
return;
}
+try {
+ if (!rpm.getQueueStorage().hasData()) {
+return;
+ }
+} catch (ReplicationException e) {
+ LOG.error("Error occurred while executing queueStorage.hasData()", e);
+ return;
+}
canFilter = rpm.getReplicationLogCleanerBarrier().start();
if (canFilter) {
notFullyDeadServers = getNotFullyDeadServers.get();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
index 7a227fb0603..7edadae03b1 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
@@ -86,6 +86,7 @@ public class TestReplicationLogCleaner {
when(rpm.listPeers(null)).thenReturn(new ArrayList<>());
ReplicationQueueStorage rqs = mock(ReplicationQueueStorage.class);
when(rpm.getQueueStorage()).thenReturn(rqs);
+when(rpm.getQueueStorage().hasData()).thenReturn(true);
when(rqs.listAllQueues()).thenReturn(new ArrayList<>());
ServerManager sm = mock(ServerManager.class);
when(services.getServerManager()).thenReturn(sm);
[hbase] 02/10: HBASE-27213 Add support for claim queue operation (#4708)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4a75db14b7a6c70fa1dfaffdf3a4e37e88b9de57
Author: Duo Zhang
AuthorDate: Sat Aug 20 23:10:58 2022 +0800
HBASE-27213 Add support for claim queue operation (#4708)
Signed-off-by: Xin Sun
---
.../protobuf/server/master/MasterProcedure.proto | 6 +--
.../AssignReplicationQueuesProcedure.java | 13 ++---
.../master/replication/ModifyPeerProcedure.java| 2 +-
.../master/replication/RemovePeerProcedure.java| 41 +-
.../regionserver/ReplicationSourceManager.java | 37 +
.../replication/TestClaimReplicationQueue.java | 2 +-
...java => TestRemovePeerProcedureWaitForSCP.java} | 63 +-
.../replication/TestSerialReplicationFailover.java | 3 --
8 files changed, 116 insertions(+), 51 deletions(-)
diff --git
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 2e0da0deb84..76a1d676487 100644
---
a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++
b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -515,6 +515,7 @@ message UpdatePeerConfigStateData {
message RemovePeerStateData {
optional ReplicationPeer peer_config = 1;
+ repeated int64 ongoing_assign_replication_queues_proc_ids = 2;
}
message EnablePeerStateData {
@@ -714,9 +715,8 @@ message ModifyColumnFamilyStoreFileTrackerStateData {
}
enum AssignReplicationQueuesState {
- ASSIGN_REPLICATION_QUEUES_PRE_CHECK = 1;
- ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 2;
- ASSIGN_REPLICATION_QUEUES_CLAIM = 3;
+ ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 1;
+ ASSIGN_REPLICATION_QUEUES_CLAIM = 2;
}
message AssignReplicationQueuesStateData {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
index e7fb5e51715..d33259dd436 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
@@ -102,8 +103,12 @@ public class AssignReplicationQueuesProcedure
}
private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException
{
+Set existingPeerIds =
env.getReplicationPeerManager().listPeers(null).stream()
+ .map(ReplicationPeerDescription::getPeerId).collect(Collectors.toSet());
ReplicationQueueStorage storage =
env.getReplicationPeerManager().getQueueStorage();
-List queueIds = storage.listAllQueueIds(crashedServer);
+// filter out replication queue for deleted peers
+List queueIds =
storage.listAllQueueIds(crashedServer).stream()
+ .filter(q ->
existingPeerIds.contains(q.getPeerId())).collect(Collectors.toList());
if (queueIds.isEmpty()) {
LOG.debug("Finish claiming replication queues for {}", crashedServer);
// we are done
@@ -130,10 +135,6 @@ public class AssignReplicationQueuesProcedure
throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
try {
switch (state) {
-case ASSIGN_REPLICATION_QUEUES_PRE_CHECK:
- // TODO: reserved for implementing the fencing logic with
Add/Remove/UpdatePeerProcedure
-
setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES);
- return Flow.HAS_MORE_STATE;
case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
addMissingQueues(env);
retryCounter = null;
@@ -183,7 +184,7 @@ public class AssignReplicationQueuesProcedure
@Override
protected AssignReplicationQueuesState getInitialState() {
-return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_PRE_CHECK;
+return
AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES;
}
@Override
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 67d70a166be..78b97620c01 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++
[hbase] 08/10: HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d2c82c7750d3720665a2f2f3eece2c15bcbeb7b0
Author: LiangJun He <2005hit...@163.com>
AuthorDate: Sun Nov 13 22:03:36 2022 +0800
HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
Signed-off-by: Duo Zhang
---
.../regionserver/DumpReplicationQueues.java| 240 +
.../hadoop/hbase/wal/AbstractFSWALProvider.java| 20 ++
.../regionserver/TestDumpReplicationQueues.java| 159 +-
3 files changed, 284 insertions(+), 135 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 98d0a55fbc4..b284e3f6837 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -19,8 +19,12 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -31,7 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Admin;
@@ -40,28 +44,33 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.replication.TableCFs;
import org.apache.hadoop.hbase.io.WALLink;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationOffsetUtil;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.zookeeper.ZKDump;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import
org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
- * TODO: reimplement this tool
*
* Provides information about the existing states of replication, replication
peers and queues.
* Usage: hbase
org.apache.hadoop.hbase.replication.regionserver.DumpReplicationQueues [args]
* Arguments: --distributed Polls each RS to dump information about the queue
--hdfs Reports HDFS
- * usage by the replication queues (note: can be overestimated).
+ * usage by the replication queues (note: can be overestimated). In the new
version, we
+ * reimplemented the DumpReplicationQueues tool to support obtaining
information from replication
+ * table.
*/
@InterfaceAudience.Private
public class DumpReplicationQueues extends Configured implements Tool {
@@ -185,7 +194,7 @@ public class DumpReplicationQueues extends Configured
implements Tool {
System.err.println("General Options:");
System.err.println(" -h|--h|--help Show this help and exit.");
System.err.println(" --distributed Poll each RS and print its own
replication queue. "
- + "Default only polls ZooKeeper");
+ + "Default only polls replication table.");
System.err.println(" --hdfs Use HDFS to calculate usage of WALs by
replication."
+ " It could be overestimated if replicating to multiple peers."
+ " --distributed flag is also needed.");
@@ -201,13 +210,7 @@ public class DumpReplicationQueues extends Configured
implements Tool {
Connection connection = ConnectionFactory.createConnection(conf);
Admin
[hbase] 04/10: HBASE-27215 Add support for sync replication (#4762)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4f5db83ac61d561afa864f9709e17cfd07d61b6c
Author: Duo Zhang
AuthorDate: Thu Sep 15 22:58:29 2022 +0800
HBASE-27215 Add support for sync replication (#4762)
Signed-off-by: Xiaolin Ha
---
.../regionserver/ReplicationSource.java| 2 +-
.../regionserver/ReplicationSourceManager.java | 53 +++---
.../TestDrainReplicationQueuesForStandBy.java | 3 --
3 files changed, 28 insertions(+), 30 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 45b66bcb1dc..788fb4871c9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -465,7 +465,7 @@ public class ReplicationSource implements
ReplicationSourceInterface {
t.getName());
manager.refreshSources(peerId);
break;
-} catch (IOException e1) {
+} catch (IOException | ReplicationException e1) {
LOG.error("Replication sources refresh failed.", e1);
sleepForRetries("Sleeping before try refreshing sources again",
maxRetriesMultiplier);
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 3397fffc036..c887988fd92 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -403,38 +403,44 @@ public class ReplicationSourceManager {
// TODO: use empty initial offsets for now, revisit when adding support
for sync replication
ReplicationSourceInterface src =
createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
-// synchronized here to avoid race with preLogRoll where we add new log to
source and also
+// synchronized here to avoid race with postLogRoll where we add new log
to source and also
// walsById.
ReplicationSourceInterface toRemove;
-Map> wals = new HashMap<>();
+ReplicationQueueData queueData;
synchronized (latestPaths) {
+ // Here we make a copy of all the remaining wal files and then delete
them from the
+ // replication queue storage after releasing the lock. It is not safe to
just remove the old
+ // map from walsById since later we may fail to update the replication
queue storage, and when
+ // we retry next time, we can not know the wal files that needs to be
set to the replication
+ // queue storage
+ ImmutableMap.Builder builder =
ImmutableMap.builder();
+ synchronized (walsById) {
+walsById.get(queueId).forEach((group, wals) -> {
+ if (!wals.isEmpty()) {
+builder.put(group, new ReplicationGroupOffset(wals.last(), -1));
+ }
+});
+ }
+ queueData = new ReplicationQueueData(queueId, builder.build());
+ src = createSource(queueData, peer);
toRemove = sources.put(peerId, src);
if (toRemove != null) {
LOG.info("Terminate replication source for " + toRemove.getPeerId());
toRemove.terminate(terminateMessage);
toRemove.getSourceMetrics().clear();
}
- // Here we make a copy of all the remaining wal files and then delete
them from the
- // replication queue storage after releasing the lock. It is not safe to
just remove the old
- // map from walsById since later we may fail to delete them from the
replication queue
- // storage, and when we retry next time, we can not know the wal files
that need to be deleted
- // from the replication queue storage.
- walsById.get(queueId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
+}
+for (Map.Entry entry :
queueData.getOffsets().entrySet()) {
+ queueStorage.setOffset(queueId, entry.getKey(), entry.getValue(),
Collections.emptyMap());
}
LOG.info("Startup replication source for " + src.getPeerId());
src.startup();
-for (NavigableSet walsByGroup : wals.values()) {
- // TODO: just need to reset the replication offset
- // for (String wal : walsByGroup) {
- // queueStorage.removeWAL(server.getServerName(), peerId, wal);
- // }
-}
synchronized (walsById) {
- Map> oldWals = walsById.get(queueId);
- wals.forEach((k, v) -> {
-NavigableSet walsByGroup = oldWals.get(k);
+ Map>
