tkalkirill commented on code in PR #4361: URL: https://github.com/apache/ignite-3/pull/4361#discussion_r1754162434
########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { + Node node0 = startNode(); + Node node1 = startNode(); + Node node2 = startNode(); + + node0.cmgManager.initCluster(List.of(node0.name(), node1.name(), node2.name()), List.of(node0.name()), "test"); + + assertThat( + allOf(node0.cmgManager.onJoinReady(), node1.cmgManager.onJoinReady(), node2.cmgManager.onJoinReady()), + willCompleteSuccessfully() + ); + + node0.waitWatches(); + node1.waitWatches(); + node2.waitWatches(); + } + + private static void assertThatMetastorageHasNoMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willTimeoutIn(1, SECONDS)); + } + + private static void assertThatMetastorageHasMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willCompleteSuccessfully()); + } + + private void stopAllNodesBut0() { + for (int i = 1; i < nodes.size(); i++) { + Node node = nodes.get(i); + node.clusterService.beforeNodeStop(); + assertThat(node.clusterService.stopAsync(), willCompleteSuccessfully()); + } + } + + @Test + void becomeLonelyLeaderStopsLearnerManagementIfPauseRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertFalse( Review Comment: Leave a comment on what we expect here and why. ########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { Review Comment: Why not just call it "start3Nodes"? the name confused me a bit, I had to look into the implementation. ########## modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/impl/MetaStorageManagerImpl.java: ########## @@ -872,6 +884,24 @@ public CompletableFuture<IndexWithTerm> raftNodeIndex() { })); } + @Override + public CompletableFuture<Void> becomeLonelyLeader(boolean pauseLeaderSecondaryDuties) { Review Comment: Can this method be called in parallel? If not, then everything is fine. If yes, then we need to protect it somehow. ########## modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/impl/MetaStorageLeaderElectionListener.java: ########## @@ -94,7 +98,8 @@ public class MetaStorageLeaderElectionListener implements LeaderElectionListener CompletableFuture<MetaStorageServiceImpl> metaStorageSvcFut, ClusterTimeImpl clusterTime, CompletableFuture<MetaStorageConfiguration> metaStorageConfigurationFuture, - List<ElectionListener> electionListeners + List<ElectionListener> electionListeners, + BooleanSupplier leaderSecondaryDutiesPaused Review Comment: Can you give a description somewhere of what this supplier is for? ########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { + Node node0 = startNode(); + Node node1 = startNode(); + Node node2 = startNode(); + + node0.cmgManager.initCluster(List.of(node0.name(), node1.name(), node2.name()), List.of(node0.name()), "test"); + + assertThat( + allOf(node0.cmgManager.onJoinReady(), node1.cmgManager.onJoinReady(), node2.cmgManager.onJoinReady()), + willCompleteSuccessfully() + ); + + node0.waitWatches(); + node1.waitWatches(); + node2.waitWatches(); + } + + private static void assertThatMetastorageHasNoMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willTimeoutIn(1, SECONDS)); + } + + private static void assertThatMetastorageHasMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willCompleteSuccessfully()); + } + + private void stopAllNodesBut0() { + for (int i = 1; i < nodes.size(); i++) { + Node node = nodes.get(i); + node.clusterService.beforeNodeStop(); + assertThat(node.clusterService.stopAsync(), willCompleteSuccessfully()); + } + } + + @Test + void becomeLonelyLeaderStopsLearnerManagementIfPauseRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertFalse( + waitForCondition(() -> learnersAt(node0).contains(node3.name()), SECONDS.toMillis(3)), + "The leader still manages learners" + ); + } + + @Test + void becomeLonelyLeaderContinuesLearnerManagementIfPauseNotRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(false), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertTrue( Review Comment: Same ########## modules/system-disaster-recovery/src/main/java/org/apache/ignite/internal/disaster/system/SystemDisasterRecoveryManagerImpl.java: ########## @@ -334,6 +356,26 @@ private static boolean isMajorityOfCmgAreSuccesses( return successes >= (futuresFromNewCmg.size() + 1) / 2; } + private static String errorMessageForNotEnoughSuccesses( + boolean repairMg, + Map<String, CompletableFuture<NetworkMessage>> responseFutures + ) { + if (repairMg) { + return String.format( + "Did not get successful response from at least one node, failing cluster reset [failedNode=%s].", + findAnyFailedNodeName(responseFutures) + ); + } else { + return "Did not get successful responses from new CMG majority, failing cluster reset."; + } + } + + private static String findAnyFailedNodeName(Map<String, CompletableFuture<NetworkMessage>> responseFutures) { + return responseFutures.entrySet().stream() + .filter(entry -> entry.getValue().isCompletedExceptionally()) + .findAny().orElseThrow().getKey(); Review Comment: Because of "orElseThrow()" we won't get a weird error? ########## modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/impl/MetaStorageLeaderElectionListener.java: ########## @@ -179,6 +197,12 @@ private interface Action { * Executes the given action if the current node is the Meta Storage leader. */ private void execute(Action action) { + if (leaderSecondaryDutiesPaused.getAsBoolean()) { Review Comment: Why not inside the busyLock? ########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { + Node node0 = startNode(); + Node node1 = startNode(); + Node node2 = startNode(); + + node0.cmgManager.initCluster(List.of(node0.name(), node1.name(), node2.name()), List.of(node0.name()), "test"); + + assertThat( + allOf(node0.cmgManager.onJoinReady(), node1.cmgManager.onJoinReady(), node2.cmgManager.onJoinReady()), + willCompleteSuccessfully() + ); + + node0.waitWatches(); + node1.waitWatches(); + node2.waitWatches(); + } + + private static void assertThatMetastorageHasNoMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willTimeoutIn(1, SECONDS)); + } + + private static void assertThatMetastorageHasMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willCompleteSuccessfully()); + } + + private void stopAllNodesBut0() { + for (int i = 1; i < nodes.size(); i++) { + Node node = nodes.get(i); + node.clusterService.beforeNodeStop(); + assertThat(node.clusterService.stopAsync(), willCompleteSuccessfully()); + } + } + + @Test + void becomeLonelyLeaderStopsLearnerManagementIfPauseRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertFalse( + waitForCondition(() -> learnersAt(node0).contains(node3.name()), SECONDS.toMillis(3)), + "The leader still manages learners" + ); + } + + @Test + void becomeLonelyLeaderContinuesLearnerManagementIfPauseNotRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(false), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertTrue( + waitForCondition(() -> learnersAt(node0).contains(node3.name()), SECONDS.toMillis(10)), + "The leader does not manage learners" + ); + } + + private static Set<String> learnersAt(Node node0) { + CompletableFuture<Set<String>> future = node0.getMetaStorageLearners(); + + assertThat(future, willCompleteSuccessfully()); + + return future.join(); + } + + @Test + void becomeLonelyLeaderStopsIdleSafeTimePropagationIfPauseRequested() throws Exception { + enableIdleSafeTimeSync(); + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + ClusterTime clusterTime0 = node0.metaStorageManager.clusterTime(); + HybridTimestamp timeBeforeOp = clusterTime0.currentSafeTime(); + + assertFalse( + waitForCondition(() -> clusterTime0.currentSafeTime().longValue() > timeBeforeOp.longValue(), SECONDS.toMillis(2)), + "The leader still propagates safetime" + ); + } + + @Test + void becomeLonelyLeaderKeepsIdleSafeTimePropagationIfPauseNotRequested() throws Exception { + enableIdleSafeTimeSync(); + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(false), willCompleteSuccessfully()); + + ClusterTime clusterTime0 = node0.metaStorageManager.clusterTime(); + HybridTimestamp timeBeforeOp = clusterTime0.currentSafeTime(); + + assertTrue( Review Comment: same ########## modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/impl/MetastorageGroupMaintenance.java: ########## @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.raft.IndexWithTerm; + +/** + * Entry point for tasks related to maintenance of the Metastorage Raft group. + */ Review Comment: ```suggestion /** Entry point for tasks related to maintenance of the Metastorage Raft group. */ ``` ########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { + Node node0 = startNode(); + Node node1 = startNode(); + Node node2 = startNode(); + + node0.cmgManager.initCluster(List.of(node0.name(), node1.name(), node2.name()), List.of(node0.name()), "test"); + + assertThat( + allOf(node0.cmgManager.onJoinReady(), node1.cmgManager.onJoinReady(), node2.cmgManager.onJoinReady()), + willCompleteSuccessfully() + ); + + node0.waitWatches(); + node1.waitWatches(); + node2.waitWatches(); + } + + private static void assertThatMetastorageHasNoMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willTimeoutIn(1, SECONDS)); + } + + private static void assertThatMetastorageHasMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willCompleteSuccessfully()); + } + + private void stopAllNodesBut0() { + for (int i = 1; i < nodes.size(); i++) { + Node node = nodes.get(i); + node.clusterService.beforeNodeStop(); + assertThat(node.clusterService.stopAsync(), willCompleteSuccessfully()); + } + } + + @Test + void becomeLonelyLeaderStopsLearnerManagementIfPauseRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertFalse( + waitForCondition(() -> learnersAt(node0).contains(node3.name()), SECONDS.toMillis(3)), + "The leader still manages learners" + ); + } + + @Test + void becomeLonelyLeaderContinuesLearnerManagementIfPauseNotRequested() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(false), willCompleteSuccessfully()); + + Node node3 = startNode(); + + assertTrue( + waitForCondition(() -> learnersAt(node0).contains(node3.name()), SECONDS.toMillis(10)), + "The leader does not manage learners" + ); + } + + private static Set<String> learnersAt(Node node0) { + CompletableFuture<Set<String>> future = node0.getMetaStorageLearners(); + + assertThat(future, willCompleteSuccessfully()); + + return future.join(); + } + + @Test + void becomeLonelyLeaderStopsIdleSafeTimePropagationIfPauseRequested() throws Exception { + enableIdleSafeTimeSync(); + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Stop the majority. + stopAllNodesBut0(); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + ClusterTime clusterTime0 = node0.metaStorageManager.clusterTime(); + HybridTimestamp timeBeforeOp = clusterTime0.currentSafeTime(); + + assertFalse( Review Comment: same ########## modules/metastorage/src/integrationTest/java/org/apache/ignite/internal/metastorage/impl/ItMetaStorageMaintenanceTest.java: ########## @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.metastorage.impl; + +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutIn; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; +import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.apache.ignite.internal.lang.ByteArray; +import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.metastorage.server.time.ClusterTime; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ConfigurationExtension.class) +class ItMetaStorageMaintenanceTest extends ItMetaStorageMultipleNodesAbstractTest { + @Test + void becomeLonelyLeaderMakesNodeLeaderForcefully() throws Exception { + start3VotingNodes(); + + Node node0 = nodes.get(0); + + // Metastorage works. + assertThatMetastorageHasMajority(node0); + + // Stop the majority. + stopAllNodesBut0(); + + // Metastorage does not work anymore. + assertThatMetastorageHasNoMajority(node0); + + assertThat(node0.metaStorageManager.becomeLonelyLeader(true), willCompleteSuccessfully()); + + assertThatMetastorageHasMajority(node0); + } + + private void start3VotingNodes() throws NodeStoppingException { + Node node0 = startNode(); + Node node1 = startNode(); + Node node2 = startNode(); + + node0.cmgManager.initCluster(List.of(node0.name(), node1.name(), node2.name()), List.of(node0.name()), "test"); + + assertThat( + allOf(node0.cmgManager.onJoinReady(), node1.cmgManager.onJoinReady(), node2.cmgManager.onJoinReady()), + willCompleteSuccessfully() + ); + + node0.waitWatches(); + node1.waitWatches(); + node2.waitWatches(); + } + + private static void assertThatMetastorageHasNoMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willTimeoutIn(1, SECONDS)); + } + + private static void assertThatMetastorageHasMajority(Node node0) { + assertThat(node0.metaStorageManager.get(new ByteArray("abc")), willCompleteSuccessfully()); + } + + private void stopAllNodesBut0() { Review Comment: Maybe rename to "stopAllNodesExcept0"? ########## modules/system-disaster-recovery-api/src/main/java/org/apache/ignite/internal/disaster/system/message/BecomeMetastorageLeaderMessage.java: ########## @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.disaster.system.message; + +import org.apache.ignite.internal.network.NetworkMessage; +import org.apache.ignite.internal.network.annotations.Transferable; + +/** + * A command to make a node become a Metastorage Raft group leader. + */ Review Comment: ```suggestion /** A command to make a node become a Metastorage Raft group leader. */ ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
