Alexey Kukushkin created IGNITE-28874:
-----------------------------------------
Summary: Ignite service is never redeployed when its node fails
Key: IGNITE-28874
URL: https://issues.apache.org/jira/browse/IGNITE-28874
Project: Ignite
Issue Type: Bug
Affects Versions: 2.18
Reporter: Alexey Kukushkin
It is expected that an Ignite service is always redeployed on another available
node after its node fails. However, it does not happen if the next node that
Ignite chooses for the service also fails during the service redeployment. See
the reproducer below.
{code:java}
package sandbox.ignite;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.failure.NoOpFailureHandler;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.services.Service;
import org.apache.ignite.services.ServiceConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.junit.jupiter.api.RepeatedTest;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class CfAll44439Test {
private final static int FAILURE_DETECTION_TIMEOUT = 2_000;
/**
* EXPECTED: An Ignite service is always redeployed on the next available
node after its node fails.
* ACTUAL: The service is never redeployed if the next available node
chosen for it fails during
* the service redeployment.
*/
@RepeatedTest(10)
public void serviceIsRedeployedAfterItsNodeFails() throws
InterruptedException {
// GIVEN an Ignite cluster consisting of 1 server and 3 client nodes
with a 2-second failure detection timeout
var aliveClients = new HashSet<>(Set.of("client1", "client2",
"client3"));
try (final var server1 =
Ignition.start(getIgniteConfiguration("server1", false))) {
for (final var clientName : aliveClients) {
Ignition.start(getIgniteConfiguration(clientName, true));
}
// AND an Ignite service named "SingletonService" is deployed on
one of the client nodes as a Cluster
// Singleton. The "SingletonService" is initially deployed without
any delays. However, redeploying
// the service takes 4 times the failure detection timeout (8
seconds).
final var service = new SingletonService(Duration.ofMillis(4 *
FAILURE_DETECTION_TIMEOUT));
final var serviceName = SingletonService.class.getName();
final var serviceConfig = new ServiceConfiguration()
.setService(service)
.setName(serviceName)
.setNodeFilter(new AllClientsPredicate())
.setMaxPerNodeCount(0)
.setTotalCount(1);
server1.services().deploy(serviceConfig);
final var initialServiceNode = getServiceNodeConsistentId(server1,
serviceName);
assertTrue(
initialServiceNode.isPresent(),
"Initial setup: all clients are alive, the service is on " +
initialServiceNode);
// AND the node hosting the "SingletonService" stops
Ignition.stop(initialServiceNode.orElseThrow(), true);
aliveClients.remove(initialServiceNode.orElseThrow());
var nextServiceNode = getServiceNodeConsistentId(server1,
serviceName);
assertTrue(
nextServiceNode.isEmpty(),
"The service is not deployed immediately after its node is
killed since redeployment takes " +
"(4 x timeout)");
// AND 4 seconds passes
Thread.sleep(2 * FAILURE_DETECTION_TIMEOUT);
nextServiceNode = getServiceNodeConsistentId(server1, serviceName);
assertTrue(
nextServiceNode.isEmpty(),
"The service is still not deployed after (2 x timeout) since
redeployment takes (4 x timeout)");
// AND the next node Ignite scheduled to deploy the service on stops
// NOTE: at this moment the service is still not redeployed, but
Ignite has already chosen the next
// node for deployment. There is no API to query that node, so we
pick one of the two remaining
// client nodes at random. This means there is a 50% chance of
picking the wrong node, and thus
// the test reproduces the problem with 50% probability. The test
is repeated to increase that probability.
var nextKilledClient = aliveClients.iterator().next();
Ignition.stop(nextKilledClient, true);
aliveClients.remove(nextKilledClient);
nextServiceNode = getServiceNodeConsistentId(server1, serviceName);
assertTrue(
nextServiceNode.isEmpty(),
"The service is still not deployed immediately after another
node is killed");
// WHEN another 12 seconds pass, making the total time 16 seconds,
which is twice the
// service's 8-second redeployment time.
Thread.sleep(6 * FAILURE_DETECTION_TIMEOUT);
// THEN the service is redeployed on the remaining client node
nextServiceNode = getServiceNodeConsistentId(server1, serviceName);
final var remainingClient = aliveClients.iterator().next();
assertTrue(
nextServiceNode.isPresent() &&
nextServiceNode.orElseThrow().equals(remainingClient),
"The service must be deployed on the remaining " +
remainingClient + " after (8 x timeout) since " +
"the elapsed time significantly exceeds the service's (4 x
timeout) redeployment time");
Ignition.stop(remainingClient, true);
}
}
private static Optional<String> getServiceNodeConsistentId(Ignite ignite,
String serviceName) {
return ignite.services().serviceDescriptors().stream()
.filter(s -> s.name().equals(serviceName))
.findFirst()
.flatMap(descriptor -> descriptor
.topologySnapshot()
.keySet()
.stream()
.findFirst()
.flatMap(uuid ->
Optional.ofNullable(ignite.cluster().node(uuid)))
.map(node -> node.consistentId().toString()));
}
private static IgniteConfiguration getIgniteConfiguration(final String
igniteName, final boolean isClient) {
return new IgniteConfiguration()
.setClientMode(isClient)
.setIgniteInstanceName(igniteName)
.setConsistentId(igniteName)
.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT)
.setClientFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT)
.setFailureHandler(new NoOpFailureHandler())
.setDiscoverySpi(
new TcpDiscoverySpi()
.setIpFinder(new
TcpDiscoveryVmIpFinder().setAddresses(Collections.singleton("127.0.0.1:49500")))
.setLocalPort(49500)
);
}
private static class SingletonService implements Service {
private static boolean initiallyDeployed = false;
private final Duration redeployDuration;
private SingletonService(final Duration redeployDuration) {
this.redeployDuration = redeployDuration;
}
@Override
public void init() {
try {
if (initiallyDeployed) {
Thread.sleep(redeployDuration);
} else {
initiallyDeployed = true;
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
private static class AllClientsPredicate implements
IgnitePredicate<ClusterNode> {
@Override
public boolean apply(final ClusterNode clusterNode) {
return clusterNode.isClient();
}
}
}
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
