This is an automated email from the ASF dual-hosted git repository.
petrov-mg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 04286dff940 IGNITE-28917 Implemented tracking of previous cluster
version feature set and sending it to joining nodes (#13393)
04286dff940 is described below
commit 04286dff9401ae4e8a0c8ed31082e945ccaab4a0
Author: Mikhail Petrov <[email protected]>
AuthorDate: Thu Jul 23 16:26:07 2026 +0300
IGNITE-28917 Implemented tracking of previous cluster version feature set
and sending it to joining nodes (#13393)
---
.../rollingupgrade/RollingUpgradeClusterData.java | 22 +++++-
.../rollingupgrade/RollingUpgradeProcessor.java | 11 +--
.../feature/IgniteFeatureManager.java | 40 ++++++++--
.../rollingupgrade/AbstractRollingUpgradeTest.java | 86 ++++++++++++++++++++--
.../CoreVersionRollingUpgradeTest.java | 27 ++++---
.../PluginVersionRollingUpgradeTest.java | 6 ++
6 files changed, 163 insertions(+), 29 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java
index f9a7b491d12..1c86dbda18f 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java
@@ -20,7 +20,9 @@ package org.apache.ignite.internal.processors.rollingupgrade;
import java.util.UUID;
import org.apache.ignite.internal.Order;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatureSet;
+import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteNodeFeatureSet;
import org.apache.ignite.plugin.extensions.communication.Message;
+import org.jetbrains.annotations.Nullable;
/** */
public class RollingUpgradeClusterData implements Message {
@@ -36,6 +38,10 @@ public class RollingUpgradeClusterData implements Message {
@Order(2)
IgniteComponentFeatureSet[] activeFeatures;
+ /** */
+ @Order(3)
+ @Nullable IgniteComponentFeatureSet[] prevActiveFeatures;
+
/** */
public RollingUpgradeClusterData() {
// No-op.
@@ -45,10 +51,22 @@ public class RollingUpgradeClusterData implements Message {
public RollingUpgradeClusterData(
boolean isVersionUpgradeEnabled,
UUID curFinalizeProcId,
- IgniteComponentFeatureSet[] activeFeatures
+ IgniteNodeFeatureSet activeFeatures,
+ @Nullable IgniteNodeFeatureSet prevActiveFeatures
) {
this.isVersionUpgradeEnabled = isVersionUpgradeEnabled;
this.curFinalizeProcId = curFinalizeProcId;
- this.activeFeatures = activeFeatures;
+ this.activeFeatures = activeFeatures.values();
+ this.prevActiveFeatures = prevActiveFeatures == null ? null :
prevActiveFeatures.values();
+ }
+
+ /** */
+ public IgniteNodeFeatureSet activeFeatures() {
+ return new IgniteNodeFeatureSet(activeFeatures);
+ }
+
+ /** */
+ @Nullable public IgniteNodeFeatureSet previousActiveFeatures() {
+ return prevActiveFeatures == null ? null : new
IgniteNodeFeatureSet(prevActiveFeatures);
}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java
index d334167d2c8..76318e8a3dd 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java
@@ -218,12 +218,12 @@ public class RollingUpgradeProcessor extends
GridProcessorAdapter implements Dis
/** {@inheritDoc} */
@Override public void
onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
- RollingUpgradeClusterData gridData = data.commonData();
+ RollingUpgradeClusterData clusterData = data.commonData();
- isVerUpgradeEnabled = gridData.isVersionUpgradeEnabled;
- curFinalizeProcId = gridData.curFinalizeProcId;
+ isVerUpgradeEnabled = clusterData.isVersionUpgradeEnabled;
+ curFinalizeProcId = clusterData.curFinalizeProcId;
- featureMgr.onGridDataReceived(new
IgniteNodeFeatureSet(gridData.activeFeatures));
+ featureMgr.onGridDataReceived(clusterData);
}
/** {@inheritDoc} */
@@ -396,7 +396,8 @@ public class RollingUpgradeProcessor extends
GridProcessorAdapter implements Dis
return new RollingUpgradeClusterData(
isVerUpgradeEnabled,
curFinalizeProcId,
- featureMgr.activeFeatures().values()
+ featureMgr.activeFeatures(),
+ featureMgr.previousActiveFeatures()
);
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java
index c836bd5bbb7..af63c1be2ed 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Collection;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.GridKernalContext;
+import
org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeClusterData;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
@@ -35,12 +36,25 @@ public class IgniteFeatureManager {
/** */
private final IgniteNodeFeatureSet locVerFeatures;
- /** */
- private final GridFutureAdapter<Void> locVerFeaturesActivationFut;
+ /**
+ * During the RU process, updated nodes operate in accordance with both
the old logical version (prior to RU completion)
+ * and the new one (after RU completion). This variable stores the
features of the previous version under which this
+ * node operated.
+ *
+ * <p>
+ * If a node joins the cluster after the Rolling Upgrade has been
finalized, this value is inherited from the
+ * existing cluster nodes (after Rolling Upgrade finalization, all
cluster nodes are guaranteed to hold the same
+ * value for this field).
+ *</p>
+ */
+ @Nullable private volatile IgniteNodeFeatureSet prevActiveFeatures;
/** */
private volatile IgniteNodeFeatureSet activeFeatures;
+ /** */
+ private final GridFutureAdapter<Void> locVerFeaturesActivationFut;
+
/** */
public IgniteFeatureManager(GridKernalContext ctx, IgniteCoreFeatureSet
coreFeatures) {
this.ctx = ctx;
@@ -67,6 +81,11 @@ public class IgniteFeatureManager {
return finalActiveFeatures;
}
+ /** @return The feature set corresponding to the previous version under
which this node operated. */
+ @Nullable public IgniteNodeFeatureSet previousActiveFeatures() {
+ return prevActiveFeatures;
+ }
+
/** @return {@code true} if the specified {@link IgniteFeature} is active
in the cluster; {@code false} otherwise. */
public boolean isActive(IgniteFeature feature) {
final IgniteNodeFeatureSet finalActiveFeatures = activeFeatures;
@@ -91,15 +110,22 @@ public class IgniteFeatureManager {
}
/** */
- public void onGridDataReceived(IgniteNodeFeatureSet activeClusterFeatures)
{
- boolean hasSameFeatures = ctx.clientNode()
+ public void onGridDataReceived(RollingUpgradeClusterData clusterData) {
+ IgniteNodeFeatureSet activeClusterFeatures =
clusterData.activeFeatures();
+
+ boolean hasSameFeaturesAsCluster = ctx.clientNode()
? activeClusterFeatures.containsAll(locVerFeatures)
: locVerFeatures.equals(activeClusterFeatures);
- if (hasSameFeatures)
+ if (hasSameFeaturesAsCluster) {
+ prevActiveFeatures = clusterData.previousActiveFeatures();
+
activateLocalVersionFeatures();
- else
- this.activeFeatures = activeClusterFeatures;
+ }
+ else {
+ activeFeatures = activeClusterFeatures;
+ prevActiveFeatures = activeClusterFeatures;
+ }
}
/** */
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java
index 84532bef8ec..8e7148f0d4c 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java
@@ -39,6 +39,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -49,7 +50,10 @@ import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCompon
import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeature;
+import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureManager;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet;
+import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteNodeFeatureSet;
+import
org.apache.ignite.internal.processors.rollingupgrade.feature.IgnitePluginFeatureSet;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.TestIgniteReleaseFeatures_2_18_0;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginComponentFeatureSetProvider;
import
org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginFeature;
@@ -74,6 +78,7 @@ import org.jspecify.annotations.Nullable;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.ignite.events.EventType.EVT_NODE_VALIDATION_FAILED;
import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
/**
* Provides the ability to override a node's version and supported {@link
IgniteFeature}s in order to
@@ -167,6 +172,28 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
stopAllGrids();
}
+ /** {@inheritDoc} */
+ @Override protected void stopGrid(int idx) {
+ UUID stoppingNodeId = grid(idx).context().localNodeId();
+
+ super.stopGrid(idx);
+
+ // Rolling Upgrade tests are highly susceptible to the topology state.
Since the node stop procedure is asynchronous,
+ // we explicitly wait for all nodes to handle the node-left event and
update their local topology snapshots.
+ for (Ignite ignite : IgnitionEx.allGridsx()) {
+ try {
+ assertTrue(waitForCondition(
+ () ->
((IgniteEx)ignite).context().discovery().node(stoppingNodeId) == null,
+ getTestTimeout()));
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteException(e);
+ }
+ }
+ }
+
/** */
protected IgniteConfiguration getConfiguration(int idx, String ver) throws
Exception {
return getConfiguration(getTestIgniteInstanceName(idx), ver);
@@ -272,6 +299,7 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
startClientGrid(2, ver);
checkVersionUpgradeInactive(ver);
+ checkPreviousClusterFeatures(null);
}
/** */
@@ -357,6 +385,38 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
);
}
+ /** */
+ protected void checkPreviousClusterFeatures(@Nullable String expVer)
throws Exception {
+ TestVersions expVersions = expVer == null ? null :
TestVersions.parse(expVer);
+
+ IgniteCoreFeatureSet expPrevCoreFeatures = expVersions != null
+ ? new IgniteCoreFeatureSet(
+ IgniteProductVersion.fromString(expVersions.coreVersion()),
+
IgniteFeatureSet.buildFrom(readDeclaredCoreFeatures(expVersions.coreVersion())))
+ : null;
+
+ IgnitePluginFeatureSet expPrevPluginFeatures = expVersions != null &&
expVersions.containsPlugin()
+ ? new IgnitePluginFeatureSet(
+ TestPluginFeature.COMPONENT_NAME,
+ IgniteProductVersion.fromString(expVersions.pluginVersion()),
+
IgniteFeatureSet.buildFrom(readDeclaredPluginFeatures(expVersions.pluginVersion())))
+ : null;
+
+ for (Ignite ignite : Ignition.allGrids()) {
+ IgniteNodeFeatureSet prevFeatures =
ru(ignite).features().previousActiveFeatures();
+
+ if (expVersions == null)
+ assertNull(prevFeatures);
+ else {
+ assertNotNull(prevFeatures);
+ assertEquals(expPrevCoreFeatures,
prevFeatures.componentFeatures(IgniteCoreFeature.COMPONENT_NAME));
+
+ if (expVersions.containsPlugin())
+ assertEquals(expPrevPluginFeatures,
prevFeatures.componentFeatures(TestPluginFeature.COMPONENT_NAME));
+ }
+ }
+ }
+
/** */
protected void checkVersionUpgradeInactive(String expVer) throws Exception
{
checkVersionUpgradeEnabledStatus(false);
@@ -467,14 +527,23 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
/** */
protected void finalizeClusterVersion(int nodeIdx, String expVer) throws
Exception {
+ IgniteFeatureManager featureMgr = ru(nodeIdx).features();
+
+ String prevLogicalVer =
featureMgr.localVersionFeatures().equals(featureMgr.activeFeatures())
+ ? null // No actual version upgrade was performed.
+ : resolveCompoundVersion(ru(nodeIdx).features().activeFeatures());
+
+ checkPreviousClusterFeatures(prevLogicalVer);
+
ru(nodeIdx).finalizeClusterVersion();
checkVersionUpgradeInactive(expVer);
+ checkPreviousClusterFeatures(prevLogicalVer);
}
/** */
protected void restartNode(int nodeIdx) throws Exception {
- String ver = resolveNodeCompoundVersions(nodeIdx);
+ String ver = resolveNodeLocalCompoundVersion(nodeIdx);
boolean isClient = grid(nodeIdx).context().clientNode();
stopGrid(nodeIdx);
@@ -489,7 +558,7 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
/** */
protected void checkUpgradeFailed(int nodeIdx, String targetVer, String
errMsg) throws Exception {
- String srcVer = resolveNodeCompoundVersions(nodeIdx);
+ String srcVer = resolveNodeLocalCompoundVersion(nodeIdx);
boolean isClient = grid(nodeIdx).context().clientNode();
stopGrid(nodeIdx);
@@ -500,11 +569,16 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
}
/** */
- String resolveNodeCompoundVersions(int nodeIdx) {
- return
Arrays.stream(ru(nodeIdx).features().localVersionFeatures().values())
+ protected String resolveNodeLocalCompoundVersion(int nodeIdx) {
+ return
resolveCompoundVersion(ru(nodeIdx).features().localVersionFeatures());
+ }
+
+ /** */
+ protected String resolveCompoundVersion(IgniteNodeFeatureSet features) {
+ return Arrays.stream(features.values())
.sorted(Comparator.comparing(IgniteComponentFeatureSet::componentName))
.map(f -> semanticVersion(f.version()))
- .collect(Collectors.joining("|"));
+ .collect(Collectors.joining(" | "));
}
/** */
@@ -567,7 +641,7 @@ public abstract class AbstractRollingUpgradeTest extends
GridCommonAbstractTest
/** */
public TestVersions(List<String> cmpVersions) {
- assertTrue(!cmpVersions.isEmpty());
+ assertFalse(cmpVersions.isEmpty());
assertTrue(cmpVersions.size() <= 2);
this.cmpVersions.put(IgniteCoreFeature.COMPONENT_NAME,
cmpVersions.get(0));
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/CoreVersionRollingUpgradeTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/CoreVersionRollingUpgradeTest.java
index b2ab378725d..e3f65fd3565 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/CoreVersionRollingUpgradeTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/CoreVersionRollingUpgradeTest.java
@@ -54,6 +54,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
startGrid(0);
checkVersionUpgradeInactive("2.19.0");
+ checkPreviousClusterFeatures(null);
}
/** */
@@ -72,9 +73,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
public void testVersionUpgradeDisabledFinalization() throws Exception {
startCluster();
- ru(1).finalizeClusterVersion();
-
- checkVersionUpgradeInactive(TEST_DEFAULT_VER);
+ finalizeClusterVersion(1, TEST_DEFAULT_VER);
}
/** */
@@ -86,12 +85,12 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
checkVersionUpgradeInProgress(TEST_DEFAULT_VER, null);
- ru(1).finalizeClusterVersion();
-
- checkVersionUpgradeInactive(TEST_DEFAULT_VER);
+ finalizeClusterVersion(1, TEST_DEFAULT_VER);
restartNode(1);
restartNode(2);
+
+ checkPreviousClusterFeatures(null);
}
/** */
@@ -198,6 +197,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
checkJoinSuccess(4, "2.19.2", true);
checkVersionUpgradeInactive("2.19.2");
+ checkPreviousClusterFeatures(TEST_DEFAULT_VER);
}
/** */
@@ -273,13 +273,22 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.2"));
finalizeClusterVersion(1, "2.19.2");
+ restartNode(0);
+ checkPreviousClusterFeatures(TEST_DEFAULT_VER);
+
ru(1).enableVersionUpgrade();
forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.2",
"2.20.0"));
finalizeClusterVersion(1, "2.20.0");
+ restartNode(1);
+ checkPreviousClusterFeatures("2.19.2");
+
ru(1).enableVersionUpgrade();
forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.20.0",
"2.21.0"));
finalizeClusterVersion(1, "2.21.0");
+
+ restartNode(2);
+ checkPreviousClusterFeatures("2.20.0");
}
/** */
@@ -439,7 +448,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
blockingDiscovery(grid(0)).block();
- IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> finalizeClusterVersion(1, TEST_DEFAULT_VER));
+ IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> ru(1).finalizeClusterVersion());
waitForBlockedDiscoveryMessages(grid(0), 1, InitMessage.class);
@@ -482,7 +491,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
assertTrue(TestRollingUpgradeProcessor.nodeJoinValidationCompletedLatch.await(getTestTimeout(),
MILLISECONDS));
- IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> finalizeClusterVersion(1, TEST_DEFAULT_VER));
+ IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> ru(1).finalizeClusterVersion());
GridTestUtils.assertThrowsAnyCause(
log,
@@ -886,7 +895,7 @@ public class CoreVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest {
blockingDiscovery(grid(0)).block();
- IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> finalizeClusterVersion(1, TEST_DEFAULT_VER));
+ IgniteInternalFuture<Object> finalizeFut =
GridTestUtils.runAsync(() -> ru(1).finalizeClusterVersion());
waitForBlockedDiscoveryMessages(grid(0), 1, InitMessage.class);
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java
index 86671640776..0f7291a9665 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java
@@ -69,6 +69,9 @@ public class PluginVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest
finalizeClusterVersion(1, "2.19.0 | 2.0.0");
+ restartNode(1);
+ checkPreviousClusterFeatures("2.19.0 | 1.0.0");
+
ru(1).enableVersionUpgrade();
checkVersionUpgradeInProgress("2.19.0 | 2.0.0", "null | null");
@@ -76,6 +79,9 @@ public class PluginVersionRollingUpgradeTest extends
AbstractRollingUpgradeTest
forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.0 | 2.0.0",
"2.19.0 | 3.0.0"));
finalizeClusterVersion(1, "2.19.0 | 3.0.0");
+
+ restartNode(2);
+ checkPreviousClusterFeatures("2.19.0 | 2.0.0");
}
/** */
