anton-vinogradov commented on code in PR #13184: URL: https://github.com/apache/ignite/pull/13184#discussion_r3500992963
########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: graceful stop → swap libs → restart. */ + public void upgradeAndRestart(int nodeCnt) throws Exception { Review Comment: `nodeCnt` is never used in this method. ```suggestion public void upgradeAndRestart() throws Exception { ``` ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java: ########## @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.ru; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteClusterContainer; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer.LOCAL_WORK_DIR_PATH; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { + /** Consistent ID's. */ + private static final List<String> CONSISTENT_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Source version image tag, overridable via {@code -Dru.source.commit.hash}. */ + private static final String SOURCE_COMMIT_HASH = System.getProperty("ru.source.commit.hash", + "0ad4656eef09acda288cbad96f80f0138732d94a"); + + /** Upgrade mode. */ + private static final UpgradeMode UPGRADE_MODE = UpgradeMode.valueOf(System.getProperty("ru.upgrade.mode", + UpgradeMode.DOCKER.name())); + + /** Cache name. */ + private static final String CACHE_NAME = "ru-test-cache"; + + /** Local work directory. */ + private static final File LOCAL_WORK_DIR = new File(LOCAL_WORK_DIR_PATH); + + /** Local host-JVM nodes (LOCAL mode only). */ + private final List<IgniteEx> nodes = new ArrayList<>(); + + /** Consistent ID -> discovery address. */ + private final Map<String, String> addrs = new HashMap<>(); + + /** Thin client. */ + private IgniteClient client; + + /** */ + @BeforeClass + public static void beforeClass() { + U.delete(LOCAL_WORK_DIR); + } + + /** */ + @AfterClass + public static void afterClass() { + U.delete(LOCAL_WORK_DIR); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return false; + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return super.getTestTimeout() * 2; + } + + /** Basic RU test. */ + @Test + public void testRollingUpgrade() throws Exception { + try (IgniteClusterContainer cluster = new IgniteClusterContainer(SOURCE_COMMIT_HASH, CONSISTENT_IDS)) { + cluster.start(); + + ClientCacheConfiguration cfg = new ClientCacheConfiguration() + .setName(CACHE_NAME) + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + ClientCache<Integer, Integer> cache = client(cluster.containers().get(0).clientAddress()).createCache(cfg); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + closeClient(); + + upgradeCluster(cluster); + + if (UPGRADE_MODE == UpgradeMode.DOCKER) + verifyViaDockerNodes(cluster); + else + verifyViaLocalNodes(); + } + finally { + closeClient(); + + if (UPGRADE_MODE == UpgradeMode.LOCAL) + stopLocalNodes(); + } + } + + /** Verify data via local host-JVM nodes. */ + private void verifyViaLocalNodes() { + IgniteCache<Integer, Integer> targetCache = nodes.get(0).cache(CACHE_NAME); + + for (int i = 0; i < 1000; i++) + assertEquals("Data mismatch after upgrade at key: " + i, (Integer)i, targetCache.get(i)); + + targetCache.put(1001, 1001); + + assertEquals((Integer)1001, targetCache.get(1001)); + } + + /** Verify data via thin client connected to upgraded Docker nodes. */ + private void verifyViaDockerNodes(IgniteClusterContainer cluster) { + IgniteContainer con = cluster.containers().get(0); + + con.checkNodeCount(cluster.containers().size()); + + ClientCache<Integer, Integer> targetCache = client(con.clientAddress()).getOrCreateCache(CACHE_NAME); + + for (int i = 0; i < 1000; i++) + assertEquals("Data mismatch after upgrade at key: " + i, (Integer)i, targetCache.get(i)); + + targetCache.put(1001, 1001); + + assertEquals((Integer)1001, targetCache.get(1001)); + } + + /** */ + private void upgradeCluster(IgniteClusterContainer srcCluster) throws Exception { + List<IgniteContainer> srcContainers = srcCluster.containers(); + + if (UPGRADE_MODE == UpgradeMode.LOCAL) + for (IgniteContainer con : srcContainers) + addrs.put(con.consistentId(), con.discoveryAddress()); + + for (int i = 0; i < srcContainers.size(); i++) { + IgniteContainer con = srcContainers.get(i); + + log.info(">>> Upgrade node=" + con.consistentId() + " (mode=" + UPGRADE_MODE + ")"); + + if (UPGRADE_MODE == UpgradeMode.DOCKER) + con.upgradeAndRestart(srcContainers.size()); Review Comment: Follow-up to dropping the unused `nodeCnt` parameter above. ```suggestion con.upgradeAndRestart(); ``` ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java: ########## @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.ru; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteClusterContainer; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer.LOCAL_WORK_DIR_PATH; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { + /** Consistent ID's. */ + private static final List<String> CONSISTENT_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Source version image tag, overridable via {@code -Dru.source.commit.hash}. */ + private static final String SOURCE_COMMIT_HASH = System.getProperty("ru.source.commit.hash", + "0ad4656eef09acda288cbad96f80f0138732d94a"); + + /** Upgrade mode. */ + private static final UpgradeMode UPGRADE_MODE = UpgradeMode.valueOf(System.getProperty("ru.upgrade.mode", + UpgradeMode.DOCKER.name())); + + /** Cache name. */ + private static final String CACHE_NAME = "ru-test-cache"; + + /** Local work directory. */ + private static final File LOCAL_WORK_DIR = new File(LOCAL_WORK_DIR_PATH); + + /** Local host-JVM nodes (LOCAL mode only). */ + private final List<IgniteEx> nodes = new ArrayList<>(); + + /** Consistent ID -> discovery address. */ + private final Map<String, String> addrs = new HashMap<>(); + + /** Thin client. */ + private IgniteClient client; + + /** */ + @BeforeClass + public static void beforeClass() { + U.delete(LOCAL_WORK_DIR); + } + + /** */ + @AfterClass + public static void afterClass() { + U.delete(LOCAL_WORK_DIR); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return false; + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return super.getTestTimeout() * 2; + } + + /** Basic RU test. */ + @Test + public void testRollingUpgrade() throws Exception { + try (IgniteClusterContainer cluster = new IgniteClusterContainer(SOURCE_COMMIT_HASH, CONSISTENT_IDS)) { + cluster.start(); + + ClientCacheConfiguration cfg = new ClientCacheConfiguration() + .setName(CACHE_NAME) + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + ClientCache<Integer, Integer> cache = client(cluster.containers().get(0).clientAddress()).createCache(cfg); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + closeClient(); + + upgradeCluster(cluster); + + if (UPGRADE_MODE == UpgradeMode.DOCKER) + verifyViaDockerNodes(cluster); + else + verifyViaLocalNodes(); + } + finally { + closeClient(); + + if (UPGRADE_MODE == UpgradeMode.LOCAL) + stopLocalNodes(); + } + } + + /** Verify data via local host-JVM nodes. */ + private void verifyViaLocalNodes() { + IgniteCache<Integer, Integer> targetCache = nodes.get(0).cache(CACHE_NAME); + + for (int i = 0; i < 1000; i++) + assertEquals("Data mismatch after upgrade at key: " + i, (Integer)i, targetCache.get(i)); + + targetCache.put(1001, 1001); + + assertEquals((Integer)1001, targetCache.get(1001)); + } + + /** Verify data via thin client connected to upgraded Docker nodes. */ + private void verifyViaDockerNodes(IgniteClusterContainer cluster) { + IgniteContainer con = cluster.containers().get(0); + + con.checkNodeCount(cluster.containers().size()); + + ClientCache<Integer, Integer> targetCache = client(con.clientAddress()).getOrCreateCache(CACHE_NAME); + + for (int i = 0; i < 1000; i++) + assertEquals("Data mismatch after upgrade at key: " + i, (Integer)i, targetCache.get(i)); + + targetCache.put(1001, 1001); + + assertEquals((Integer)1001, targetCache.get(1001)); + } + + /** */ + private void upgradeCluster(IgniteClusterContainer srcCluster) throws Exception { + List<IgniteContainer> srcContainers = srcCluster.containers(); + + if (UPGRADE_MODE == UpgradeMode.LOCAL) + for (IgniteContainer con : srcContainers) + addrs.put(con.consistentId(), con.discoveryAddress()); + + for (int i = 0; i < srcContainers.size(); i++) { + IgniteContainer con = srcContainers.get(i); + + log.info(">>> Upgrade node=" + con.consistentId() + " (mode=" + UPGRADE_MODE + ")"); + + if (UPGRADE_MODE == UpgradeMode.DOCKER) + con.upgradeAndRestart(srcContainers.size()); + else + upgradeLocally(con, i); + } + } + + /** Stop container, start a local host-JVM node with the same consistent ID. */ + private void upgradeLocally(IgniteContainer con, int idx) throws Exception { + // Address containers use to reach this (host JVM) node: the Docker bridge gateway on Linux, the + // host.docker.internal alias on macOS. + String hostIp = IgniteContainer.LINUX + ? con.gatewayIp() + : con.execInContainer("sh", "-c", + "getent ahostsv4 host.docker.internal | awk '{print $1}' | head -1").getStdout().trim(); + + con.stop(); + + addrs.remove(con.consistentId()); + + IgniteEx ignite = startGrid(configuration(con.consistentId(), con.localWorkDirectory(), addrs.values(), hostIp, idx)); + + assertTrue("Upgraded node did not rejoin the full topology in time", + waitForCondition(() -> CONSISTENT_IDS.size() == ignite.cluster().nodes().size(), DFLT_TEST_TIMEOUT)); + + addrs.put(con.consistentId(), "127.0.0.1:" + (48500 + idx)); + + nodes.add(ignite); + } + + /** */ + private IgniteConfiguration configuration(String nodeId, String workDir, Collection<String> addrs0, String ip, int idx) { + DataRegionConfiguration dataRegionCfg = new DataRegionConfiguration() + .setName("testRegion") + .setInitialSize(1024L * 1024 * 1024) + .setMaxSize(10L * 1024 * 1024 * 1024) + .setPersistenceEnabled(true); + + TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi() + .setLocalAddress("0.0.0.0") + .setIpFinder(new TcpDiscoveryVmIpFinder().setAddresses(addrs0)) + // Short socket timeout: unreachable container-internal addresses must fail fast before the + // host-reachable 127.0.0.1:<published-port> (advertised by the containers) is tried. + .setSocketTimeout(1000) + .setNetworkTimeout(20000) + .setJoinTimeout(30000) + .setLocalPort(48500 + idx); + + // On macOS communication binds to loopback (discovery stays on 0.0.0.0 to satisfy Ignite's non-loopback + // join check) so the node advertises only 127.0.0.1 + the resolver-mapped Docker host address -- no + // unreachable host LAN IPs for the containers to stall on. A short connect timeout makes the node's + // own outgoing attempts to unreachable container-internal (172.x) addresses give up in ~1s (they + // otherwise hang in SYN_SENT) and fall through to the reachable 127.0.0.1:<published-port>. + TcpCommunicationSpi commSpi = new TcpCommunicationSpi() + // macOS: bind comm to loopback (advertised to containers via the resolver as the Docker-host address). + // Linux: bind to all interfaces so containers reach this host node at the Docker bridge gateway IP. + .setLocalAddress(IgniteContainer.LINUX ? "0.0.0.0" : "127.0.0.1") + .setLocalPort(49100 + idx) + .setConnectTimeout(1000) + .setMaxConnectTimeout(10000) + // The NIO connect to a blackholed container-internal (172.x) address is not aborted by + // connectTimeout on macOS (it hangs in SYN_SENT for the OS timeout, ~75s), stalling the exchange. + // Pre-filter unreachable addresses so only the reachable 127.0.0.1:<published-port> is used. + .setFilterReachableAddresses(true); + + return new IgniteConfiguration() + .setIgniteInstanceName(nodeId) + .setConsistentId(nodeId) + .setWorkDirectory(workDir) + .setDataStorageConfiguration(new DataStorageConfiguration().setDefaultDataRegionConfiguration(dataRegionCfg)) + .setDiscoverySpi(discoverySpi) + .setAddressResolver(addr -> { + int port = addr.getPort(); + + // Each sequentially started host node binds the next port in the discovery (48500+) and + // communication (47100+) ranges; map them all to the Docker host address so the containers Review Comment: The host node's communication range is `49100+` (`setLocalPort(49100 + idx)` a few lines above), not `47100+` — the comment is misleading. ```suggestion // communication (49100+) ranges; map them all to the Docker host address so the containers ``` ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; Review Comment: Import for the try-with-resources fix below. ```suggestion import java.util.regex.Pattern; import java.util.stream.Stream; ``` ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: graceful stop → swap libs → restart. */ + public void upgradeAndRestart(int nodeCnt) throws Exception { + stopGraceful(); + + restartWithTargetLibs(TARGET_LIBS_DIR); + + assertTrue("Upgraded Docker node is not running", isRunning()); + } + + /** + * Stop the container gracefully <b>without removing it</b> (container stays in "Exited" state). + * Call this before {@link #restartWithTargetLibs(Path)}. + * + * <p>Uses {@code docker stop} (SIGTERM + wait + SIGKILL after timeout) via the Docker API. + * This gives Ignite time to flush persistence data, and falls back to SIGKILL if needed.</p> + */ + private void stopGraceful() { + if (!isRunning()) + return; + + LOGGER.info("Graceful stop of node {}", hostname); + + getDockerClient().stopContainerCmd(getContainerId()) + .withTimeout(30) + .exec(); + + LOGGER.info("Node {} stopped", hostname); + } + + /** + * Restart the stopped container after swapping its {@code libs/} directory. + * <p> + * Copies all jars from the provided host directory into the container's {@code /opt/ignite/apache-ignite/libs/}, + * then re-injects the test-classes jar and starts the container. + * </p> + * + * @param targetLibsHostDir Host directory containing target-version jars. + */ + private void restartWithTargetLibs(Path targetLibsHostDir) throws Exception { + LOGGER.info("Replacing libs in container {} with jars from {}", hostname, targetLibsHostDir); + + for (Path file : Files.list(targetLibsHostDir).toArray(Path[]::new)) + if (Files.isRegularFile(file)) + copyFileToContainer(forHostPath(file.toAbsolutePath().toString()), LIBS_DIR_PATH + file.getFileName().toString()); Review Comment: `Files.list(...)` returns a `Stream` that holds an open directory handle and must be closed (per its javadoc). Wrap it in try-with-resources (needs the `java.util.stream.Stream` import suggested above). ```suggestion try (Stream<Path> files = Files.list(targetLibsHostDir)) { for (Path file : files.toArray(Path[]::new)) if (Files.isRegularFile(file)) copyFileToContainer(forHostPath(file.toAbsolutePath().toString()), LIBS_DIR_PATH + file.getFileName().toString()); } ``` ########## modules/compatibility/src/test/resources/docker/build_docker_image.sh: ########## @@ -0,0 +1,156 @@ +#!/bin/bash Review Comment: This script is missing the ASF license header (the sibling `build_distrib.sh` has it). RAT classifies `.sh` as binary so it won't fail the build, but ASF source files are expected to carry the header. ```suggestion #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ``` ########## modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java: ########## @@ -1269,7 +1269,8 @@ LinkedHashSet<InetSocketAddress> getEffectiveNodeAddresses(TcpDiscoveryNode node // Do not give own loopback to avoid requesting current node. if (!node.equals(locNode)) - addrs.removeIf(addr -> addr.getAddress().isLoopbackAddress() && locNode.socketAddresses().contains(addr)); + addrs.removeIf(addr -> addr.getAddress() == null || + (addr.getAddress().isLoopbackAddress() && locNode.socketAddresses().contains(addr))); Review Comment: This is the only production change in the PR, and it's a sensible NPE guard for unresolved addresses. A few asks: 1. It's currently exercised only by the Docker smoke test, which isn't part of RunAll — could you add a focused unit test for `getEffectiveNodeAddresses` with an unresolved `InetSocketAddress` (`getAddress() == null`)? 2. Dropping the address silently makes a fully-unresolvable node hard to diagnose; a `log.debug(...)` on the removed address would help. 3. Since the rest of the PR is test infrastructure, consider landing this core change as its own commit / sub-ticket so it's reviewed independently. ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java: ########## @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.ru; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteClusterContainer; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer.LOCAL_WORK_DIR_PATH; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { + /** Consistent ID's. */ + private static final List<String> CONSISTENT_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Source version image tag, overridable via {@code -Dru.source.commit.hash}. */ + private static final String SOURCE_COMMIT_HASH = System.getProperty("ru.source.commit.hash", + "0ad4656eef09acda288cbad96f80f0138732d94a"); + + /** Upgrade mode. */ + private static final UpgradeMode UPGRADE_MODE = UpgradeMode.valueOf(System.getProperty("ru.upgrade.mode", + UpgradeMode.DOCKER.name())); + + /** Cache name. */ + private static final String CACHE_NAME = "ru-test-cache"; + + /** Local work directory. */ + private static final File LOCAL_WORK_DIR = new File(LOCAL_WORK_DIR_PATH); + + /** Local host-JVM nodes (LOCAL mode only). */ + private final List<IgniteEx> nodes = new ArrayList<>(); + + /** Consistent ID -> discovery address. */ + private final Map<String, String> addrs = new HashMap<>(); + + /** Thin client. */ + private IgniteClient client; + + /** */ + @BeforeClass + public static void beforeClass() { + U.delete(LOCAL_WORK_DIR); Review Comment: The parent surefire config uses `<include>**/*.java</include>`, so a plain `mvn test` on this module will pick up this `*Test` and **fail hard** when Docker (or the pre-built image) is unavailable, instead of being skipped. Consider guarding it so it self-skips: ```java import org.junit.Assume; import org.testcontainers.DockerClientFactory; // ... @BeforeClass public static void beforeClass() { Assume.assumeTrue("Docker is required for this test", DockerClientFactory.instance().isDockerAvailable()); U.delete(LOCAL_WORK_DIR); } ``` ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java: ########## @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.ru; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteClusterContainer; +import org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.compatibility.testframework.testcontainers.IgniteContainer.LOCAL_WORK_DIR_PATH; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { + /** Consistent ID's. */ + private static final List<String> CONSISTENT_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Source version image tag, overridable via {@code -Dru.source.commit.hash}. */ + private static final String SOURCE_COMMIT_HASH = System.getProperty("ru.source.commit.hash", + "0ad4656eef09acda288cbad96f80f0138732d94a"); Review Comment: The default `ru.source.commit.hash` points at a WIP commit of this branch rather than a released version, and `apacheignite/ignite:<hash>` must be built locally first. Consider defaulting to a released version tag (or clearly documenting that this baseline is throwaway). ########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java: ########## @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.testcontainers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.plugins.DisabledRollingUpgradeProcessor; +import org.apache.ignite.compatibility.testframework.plugins.DisabledValidationProcessor; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import static org.apache.ignite.compatibility.testframework.testcontainers.ContainerAddressResolver.EXT_ADDR_PROP_PREFIX; +import static org.apache.ignite.testframework.GridTestUtils.DFLT_TEST_TIMEOUT; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.utility.MountableFile.forClasspathResource; +import static org.testcontainers.utility.MountableFile.forHostPath; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer<IgniteContainer> { + /** Local work directory. */ + public static final String LOCAL_WORK_DIR_PATH = System.getProperty("ru.local.work.dir", + U.getIgniteHome() + "/target/test-ignite-work"); + + /** + * {@code true} on Linux, where the host shares the Docker bridge and reaches containers directly. Elsewhere + * (macOS/Windows Docker Desktop) the host talks to containers through a VM proxy, so the address hacks + * (published ports + ContainerAddressResolver + host.docker.internal) are used instead. + */ + public static final boolean LINUX = System.getProperty("os.name", "").toLowerCase().contains("linux"); + + /** Host directory with target-version jars for DOCKER upgrade mode, overridable via {@code -Dru.target.libs.dir}. */ + private static final Path TARGET_LIBS_DIR = Path.of(System.getProperty("ru.target.libs.dir", + U.getIgniteHome() + "/target/ignite-target-libs")); + + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Ignite root directory in container. */ + private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; + + /** Ignite libs directory in container. */ + private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; + + /** Ignite work directory in container. */ + private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; + + /** Config path in container. */ + private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** Base host port for the published discovery port (node index added). Kept clear of the host-node ports. */ + private static final int DISCO_HOST_PORT_BASE = 50500; + + /** Base host port for the published communication port (node index added). */ + private static final int COMM_HOST_PORT_BASE = 50100; + + /** Base host port for the published thin-client port (node index added). */ + private static final int CLIENT_HOST_PORT_BASE = 50800; + + /** Custom classes (with their nested classes) used by node in containers. */ + private static final List<String> TEST_CLASSES = List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + + /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + private static volatile File testClassesJar; + + /** Hostname. */ + private final String hostname; + + /** Consistent ID. */ + private final String consistentId; + + /** Path to work directory. */ + private final String workDirPath; + + /** + * Constructor with a commit hash (image tag). + * Uses {@code apacheignite/ignite:<commitHash>} as the Docker image. + */ + public IgniteContainer(String commitHash, Network net, String hostname, String consistentId, int idx) throws IOException { + super(DockerImageName.parse("apacheignite/ignite:" + commitHash)); + + this.hostname = hostname; + this.consistentId = consistentId; + workDirPath = WORK_DIR_PATH + "/" + hostname; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Proxy-networking hosts (macOS/Windows) can't reach container-internal addresses, so each node advertises + // its host-published ports (127.0.0.1:hostPort) via ContainerAddressResolver. On Linux containers are + // directly routable and advertise their real address, so no override is needed. + if (!LINUX) { + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + } + + withEnv("JVM_OPTS", jvmOpts); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + + withNetwork(net); + withNetworkAliases(hostname); + + withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); + + // Proxy-networking hosts only: publish fixed host ports so the host JVM node can target each container at + // 127.0.0.1:<port>. On Linux the host reaches containers at their bridge IP directly, so nothing is published. + if (!LINUX) { + addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); + addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + } + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(600))); + } + + /** {@inheritDoc} */ + @Override public void stop() { + if (isRunning()) { + try { + stopGraceful(); + } + catch (Exception e) { + LOGGER.warn("Graceful shutdown failed for node {}. Proceeding with forceful stop.", hostname, e); + } + } + + super.stop(); + } + + /** In-place upgrade inside Docker: graceful stop → swap libs → restart. */ + public void upgradeAndRestart(int nodeCnt) throws Exception { + stopGraceful(); + + restartWithTargetLibs(TARGET_LIBS_DIR); + + assertTrue("Upgraded Docker node is not running", isRunning()); + } + + /** + * Stop the container gracefully <b>without removing it</b> (container stays in "Exited" state). + * Call this before {@link #restartWithTargetLibs(Path)}. + * + * <p>Uses {@code docker stop} (SIGTERM + wait + SIGKILL after timeout) via the Docker API. + * This gives Ignite time to flush persistence data, and falls back to SIGKILL if needed.</p> + */ + private void stopGraceful() { + if (!isRunning()) + return; + + LOGGER.info("Graceful stop of node {}", hostname); + + getDockerClient().stopContainerCmd(getContainerId()) + .withTimeout(30) + .exec(); + + LOGGER.info("Node {} stopped", hostname); + } + + /** + * Restart the stopped container after swapping its {@code libs/} directory. + * <p> + * Copies all jars from the provided host directory into the container's {@code /opt/ignite/apache-ignite/libs/}, + * then re-injects the test-classes jar and starts the container. + * </p> + * + * @param targetLibsHostDir Host directory containing target-version jars. + */ + private void restartWithTargetLibs(Path targetLibsHostDir) throws Exception { + LOGGER.info("Replacing libs in container {} with jars from {}", hostname, targetLibsHostDir); + + for (Path file : Files.list(targetLibsHostDir).toArray(Path[]::new)) + if (Files.isRegularFile(file)) + copyFileToContainer(forHostPath(file.toAbsolutePath().toString()), LIBS_DIR_PATH + file.getFileName().toString()); Review Comment: Note (no change required for the baseline used here): target jars are copied over the source `libs/` without removing source-only jars. For a real version gap, renamed/removed jars from the old version would linger and could cause classpath conflicts. Worth a comment or an explicit `libs/` cleanup. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
