This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 3e676e5eef9 SOLR-17879: Fail to start if its major version is smaller
than the cluster (#3510)
3e676e5eef9 is described below
commit 3e676e5eef98527821b03bbe80ba45cab0d5c66a
Author: David Smiley <[email protected]>
AuthorDate: Fri Sep 5 20:41:49 2025 -0400
SOLR-17879: Fail to start if its major version is smaller than the cluster
(#3510)
A Solr node will now fail to start if its major.minor version (e.g. 9.10)
is *lower* than that of any existing
Solr node in a SolrCloud cluster (as reported by info in "live_node").
(cherry picked from commit bd767e3f744e540530abd86cba8f4a6a8a17035c)
---
solr/CHANGES.txt | 3 +
.../java/org/apache/solr/cloud/ZkController.java | 49 +++++++
.../org/apache/solr/cloud/ZkControllerTest.java | 151 +++++++++++++++++++++
.../solr/cloud/overseer/ZkStateReaderTest.java | 32 ++++-
.../pages/major-changes-in-solr-9.adoc | 8 ++
.../apache/solr/common/cloud/ZkStateReader.java | 25 ++--
6 files changed, 253 insertions(+), 15 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 123bbfb63d3..7977e711944 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -42,6 +42,9 @@ Other Changes
* SOLR-17620: SolrCloud "live_node" now has metadata: version of Solr, roles
(Yuntong Qu, David Smiley)
+* SOLR-17879: A Solr node will now fail to start if it's major.minor version
(e.g. 9.10) is *lower* than that of any existing
+ Solr node in a SolrCloud cluster (as reported by info in "live_node").
(David Smiley)
+
================== 9.9.1 ==================
Bug Fixes
---------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index da81cc405c7..e75c6b64202 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -107,6 +107,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Compressor;
+import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectReleaseTracker;
@@ -596,6 +597,53 @@ public class ZkController implements Closeable {
}
}
+ /**
+ * Checks version compatibility with other nodes in the cluster. Refuses to
start if there's a
+ * major.minor version difference between our Solr version and other nodes
in the cluster. Note:
+ * uses live nodes.
+ */
+ private void checkClusterVersionCompatibility() throws InterruptedException,
KeeperException {
+ Optional<SolrVersion> lowestVersion =
zkStateReader.fetchLowestSolrVersion();
+ if (lowestVersion.isPresent()) {
+ SolrVersion ourVersion = SolrVersion.LATEST;
+ SolrVersion clusterVersion = lowestVersion.get();
+
+ if (ourVersion.lessThan(clusterVersion)) {
+ log.warn(
+ "Our Solr version {} is older than cluster version {}",
ourVersion, clusterVersion);
+
+ if (EnvUtils.getPropertyAsBool("solr.cloud.downgrade.enabled", false))
{
+ return;
+ }
+
+ // Check major version compatibility
+ if (ourVersion.getMajorVersion() < clusterVersion.getMajorVersion()) {
+ String message =
+ String.format(
+ Locale.ROOT,
+ "Refusing to start Solr, since our version is lower than the
lowest version currently running in the cluster. "
+ + "Our version: %s, lowest version in cluster: %s.",
+ ourVersion,
+ clusterVersion);
+ throw new SolrException(ErrorCode.INVALID_STATE, message);
+ }
+
+ // Check minor version compatibility within the same major version
+ if (ourVersion.getMajorVersion() == clusterVersion.getMajorVersion()
+ && ourVersion.getMinorVersion() <
clusterVersion.getMinorVersion()) {
+ String message =
+ String.format(
+ Locale.ROOT,
+ "Refusing to start Solr, since our version is lower than the
lowest version currently running in the cluster. "
+ + "Our version: %s, lowest version in cluster: %s.",
+ ourVersion,
+ clusterVersion);
+ throw new SolrException(ErrorCode.INVALID_STATE, message);
+ }
+ }
+ }
+ }
+
public CloudSolrClient getSolrClient() {
return getSolrCloudManager().getSolrClient();
}
@@ -983,6 +1031,7 @@ public class ZkController implements Closeable {
checkForExistingEphemeralNode();
registerLiveNodesListener();
+ checkClusterVersionCompatibility();
// start the overseer first as following code may need it's processing
if (!zkRunOnly) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
index cf78c8ff47b..2d45de8ebb3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -41,6 +41,7 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -66,6 +67,7 @@ import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.hamcrest.Matchers;
+import org.junit.Ignore;
import org.junit.Test;
@SolrTestCaseJ4.SuppressSSL
@@ -516,6 +518,155 @@ public class ZkControllerTest extends SolrCloudTestCase {
}
}
+ @Test
+ @Ignore("Would need to disable ObjectReleaseTracker")
+ public void testVersionCompatibilityFailsStartup() throws Exception {
+ Path zkDir = createTempDir("testVersionCompatibilityFailsStartup");
+ ZkTestServer server = new ZkTestServer(zkDir);
+ try {
+ server.run();
+
+ // Manually create a live node with a high version (99.0.0) to simulate
+ // a newer cluster that the current node (SolrVersion.LATEST=10.0.0)
cannot join
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(server.getZkAddress())
+ .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+
+ // Create cluster nodes first
+ ZkController.createClusterZkNodes(zkClient);
+
+ String liveNodeName = "test_node:8983_solr";
+ String liveNodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" +
liveNodeName;
+
+ // Create live node data with version 99.0.0
+ Map<String, Object> liveNodeData =
+ Map.of(LIVE_NODE_SOLR_VERSION, "99.0.0", LIVE_NODE_NODE_NAME,
liveNodeName);
+ byte[] data = Utils.toJSON(liveNodeData);
+
+ // persistent since we're about to close this zkClient
+ zkClient.create(liveNodePath, data, CreateMode.PERSISTENT, true);
+ }
+
+ // Now try to create a ZkController - this should fail due to version
incompatibility
+ CoreContainer cc = getCoreContainer();
+ try {
+ CloudConfig cloudConfig = new
CloudConfig.CloudConfigBuilder("127.0.0.1", 8984).build();
+
+ SolrException exception =
+ expectThrows(
+ SolrException.class,
+ () -> {
+ var zc = new ZkController(cc, server.getZkAddress(),
TIMEOUT, cloudConfig);
+ zc.close();
+ });
+
+ // Verify the exception is due to version incompatibility
+ assertEquals(
+ "Expected INVALID_STATE error code",
+ SolrException.ErrorCode.INVALID_STATE.code,
+ exception.code());
+ assertTrue(
+ "Exception message should mention refusing to start: " +
exception.getMessage(),
+ exception.getMessage().contains("Refusing to start Solr"));
+ assertTrue(
+ "Exception message should mention minor version: " +
exception.getMessage(),
+ exception.getMessage().contains("minor version"));
+ assertTrue(
+ "Exception message should mention our version: " +
exception.getMessage(),
+ exception.getMessage().contains("10.0.0"));
+ assertTrue(
+ "Exception message should mention cluster version: " +
exception.getMessage(),
+ exception.getMessage().contains("99.0.0"));
+ } finally {
+ cc.shutdown();
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ @Ignore("Would need to disable ObjectReleaseTracker")
+ public void testMinorVersionCompatibilityFailsStartup() throws Exception {
+ Path zkDir = createTempDir("testMinorVersionCompatibilityFailsStartup");
+ ZkTestServer server = new ZkTestServer(zkDir);
+ try {
+ server.run();
+
+ // Create a higher minor version based on SolrVersion.LATEST for cluster
simulation
+ SolrVersion currentVersion = SolrVersion.LATEST;
+ SolrVersion higherMinorVersion =
+ SolrVersion.forIntegers(
+ currentVersion.getMajorVersion(),
+ currentVersion.getMinorVersion() + 1,
+ currentVersion.getPatchVersion());
+
+ // Manually create a live node with a higher minor version to simulate
+ // a newer cluster that the current node (SolrVersion.LATEST) cannot join
+ try (SolrZkClient zkClient =
+ new SolrZkClient.Builder()
+ .withUrl(server.getZkAddress())
+ .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+
+ // Create cluster nodes first
+ ZkController.createClusterZkNodes(zkClient);
+
+ String liveNodeName = "test_node:8983_solr";
+ String liveNodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" +
liveNodeName;
+
+ // Create live node data with higher minor version (same major, higher
minor than LATEST)
+ Map<String, Object> liveNodeData =
+ Map.of(
+ LIVE_NODE_SOLR_VERSION,
+ higherMinorVersion.toString(),
+ LIVE_NODE_NODE_NAME,
+ liveNodeName);
+ byte[] data = Utils.toJSON(liveNodeData);
+
+ // persistent since we're about to close this zkClient
+ zkClient.create(liveNodePath, data, CreateMode.PERSISTENT, true);
+ }
+
+ // Now try to create a ZkController - this should fail due to minor
version incompatibility
+ CoreContainer cc = getCoreContainer();
+ try {
+ CloudConfig cloudConfig = new
CloudConfig.CloudConfigBuilder("127.0.0.1", 8984).build();
+
+ SolrException exception =
+ expectThrows(
+ SolrException.class,
+ () -> {
+ var zc = new ZkController(cc, server.getZkAddress(),
TIMEOUT, cloudConfig);
+ zc.close();
+ });
+
+ // Verify the exception is due to minor version incompatibility
+ assertEquals(
+ "Expected INVALID_STATE error code",
+ SolrException.ErrorCode.INVALID_STATE.code,
+ exception.code());
+ assertTrue(
+ "Exception message should mention refusing to start: " +
exception.getMessage(),
+ exception.getMessage().contains("Refusing to start Solr"));
+ assertTrue(
+ "Exception message should mention minor version: " +
exception.getMessage(),
+ exception.getMessage().contains("minor version"));
+ assertTrue(
+ "Exception message should mention our version: " +
exception.getMessage(),
+ exception.getMessage().contains(currentVersion.toString()));
+ assertTrue(
+ "Exception message should mention cluster version: " +
exception.getMessage(),
+ exception.getMessage().contains(higherMinorVersion.toString()));
+ } finally {
+ cc.shutdown();
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
public void testCheckNoOldClusterstate() throws Exception {
Path zkDir = createTempDir("testCheckNoOldClusterstate");
ZkTestServer server = new ZkTestServer(zkDir);
diff --git
a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java
b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java
index b9523c9b2d9..cb07e84f5b5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java
@@ -923,8 +923,9 @@ public class ZkStateReaderTest extends SolrTestCaseJ4 {
zkClient.create(livePath + "/" + node2, data2, CreateMode.EPHEMERAL, true);
var lowestVersion = reader.fetchLowestSolrVersion();
+ assertTrue("Expected lowest version to be present",
lowestVersion.isPresent());
assertEquals(
- "Expected lowest version to be 9.0.1", SolrVersion.valueOf("9.0.1"),
lowestVersion);
+ "Expected lowest version to be 9.0.1", SolrVersion.valueOf("9.0.1"),
lowestVersion.get());
}
/** Test when the only live node has empty data. */
@@ -943,7 +944,9 @@ public class ZkStateReaderTest extends SolrTestCaseJ4 {
String emptyNode = "empty_node";
zkClient.create(livePath + "/" + emptyNode, new byte[0],
CreateMode.EPHEMERAL, true);
- assertEquals("after empty node", SolrVersion.valueOf("9.9.0"),
reader.fetchLowestSolrVersion());
+ var lowestVersion = reader.fetchLowestSolrVersion();
+ assertTrue("Expected lowest version to be present for empty node",
lowestVersion.isPresent());
+ assertEquals("after empty node", SolrVersion.valueOf("9.9.0"),
lowestVersion.get());
}
/** Test when two live nodes exist; one is blank and the other has a high
version */
@@ -965,11 +968,32 @@ public class ZkStateReaderTest extends SolrTestCaseJ4 {
CreateMode.EPHEMERAL,
true);
- assertEquals("after high node", SolrVersion.LATEST,
reader.fetchLowestSolrVersion());
+ var lowestVersion1 = reader.fetchLowestSolrVersion();
+ assertTrue(
+ "Expected lowest version to be present for high version node",
lowestVersion1.isPresent());
+ assertEquals("after high node", SolrVersion.valueOf("888.0.0"),
lowestVersion1.get());
String node2 = "node2_solr";
zkClient.create(livePath + "/" + node2, new byte[0], CreateMode.EPHEMERAL,
true);
- assertEquals("after empty node", SolrVersion.valueOf("9.9.0"),
reader.fetchLowestSolrVersion());
+ var lowestVersion2 = reader.fetchLowestSolrVersion();
+ assertTrue("Expected lowest version to be present for empty node",
lowestVersion2.isPresent());
+ assertEquals("after empty node", SolrVersion.valueOf("9.9.0"),
lowestVersion2.get());
+ }
+
+ /** Test when no live nodes exist - should return empty Optional */
+ public void testFetchLowestSolrVersion_noLiveNodes() throws Exception {
+ SolrZkClient zkClient = fixture.zkClient;
+ ZkStateReader reader = fixture.reader;
+ String livePath = ZkStateReader.LIVE_NODES_ZKNODE;
+
+ // Clear any existing live node children.
+ List<String> nodes = zkClient.getChildren(livePath, null, true);
+ for (String node : nodes) {
+ zkClient.delete(livePath + "/" + node, -1, true);
+ }
+
+ var lowestVersion = reader.fetchLowestSolrVersion();
+ assertFalse("Expected no lowest version when no live nodes exist",
lowestVersion.isPresent());
}
}
diff --git
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index a220a63b8f8..9f0e1d127c8 100644
---
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -28,6 +28,14 @@ You should also consider all changes that have been made to
Solr in any version
A thorough review of the list in
xref:major-changes-in-earlier-8-x-versions[Major Changes in Earlier 8.x
Versions] as well as the {solr-javadocs}/changes/Changes.html[CHANGES.txt] in
your Solr instance will help you plan your migration to Solr 9.
+There is a new limitation introduced in Solr 9.10, and that which is
especially relevant to Solr 10
+and beyond.
+A Solr node will now fail to start if it's major.minor version (e.g. 9.10) is
*lower* than that of any existing Solr node in a SolrCloud cluster (as reported
by info in "live_node").
+What this
+means is that Solr supports rolling _upgrades_ but not rolling _downgrades_
spanning a major version.
+This compatibility safeguard can be toggled with
"SOLR_CLOUD_DOWNGRADE_ENABLED".
+
+
=== Upgrade Prerequisites
*Solr 9 requires Java 11 as minimum Java version and is also tested with Java
17.*
diff --git
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 3bff3ea21ca..24b9b1d537c 100644
---
a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++
b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -866,23 +867,25 @@ public class ZkStateReader implements SolrCloseable {
}
/**
- * Returns the lowest Solr version among all live nodes in the cluster. It's
not greater than
- * {@link SolrVersion#LATEST_STRING}. Will not return null. If older Solr
nodes have joined that
- * don't declare their version, the result won't be accurate, but it's at
least an upper bound on
- * the possible version it might be.
+ * Returns the lowest Solr version among all live nodes in the cluster. If
older Solr nodes have
+ * joined that don't declare their version, the result won't be accurate,
but it's at least an
+ * upper bound on the possible version it might be.
*
- * @return the lowest Solr version of the cluster; not null
+ * @return an Optional containing the lowest Solr version of nodes in the
cluster, or empty if no
+ * live nodes exist or all nodes return 9.9.0 for unspecified versions
*/
- public SolrVersion fetchLowestSolrVersion() throws KeeperException,
InterruptedException {
+ public Optional<SolrVersion> fetchLowestSolrVersion()
+ throws KeeperException, InterruptedException {
List<String> liveNodeNames = zkClient.getChildren(LIVE_NODES_ZKNODE, null,
true);
- SolrVersion lowest = SolrVersion.LATEST; // current software
+ SolrVersion lowest = null;
// the last version to not specify its version in live nodes
final SolrVersion UNSPECIFIED_VERSION = SolrVersion.valueOf("9.9.0");
+
for (String nodeName : liveNodeNames) {
String path = LIVE_NODES_ZKNODE + "/" + nodeName;
byte[] data = zkClient.getData(path, null, null, true);
if (data == null || data.length == 0) {
- return UNSPECIFIED_VERSION;
+ return Optional.of(UNSPECIFIED_VERSION);
}
@SuppressWarnings("unchecked")
@@ -890,14 +893,14 @@ public class ZkStateReader implements SolrCloseable {
String nodeVersionStr = (String) props.get(LIVE_NODE_SOLR_VERSION);
if (nodeVersionStr == null) { // weird
log.warn("No Solr version found: {}", props);
- return UNSPECIFIED_VERSION;
+ return Optional.of(UNSPECIFIED_VERSION);
}
SolrVersion nodeVersion = SolrVersion.valueOf(nodeVersionStr);
- if (nodeVersion.compareTo(lowest) < 0) {
+ if (lowest == null || nodeVersion.compareTo(lowest) < 0) {
lowest = nodeVersion;
}
}
- return lowest;
+ return Optional.ofNullable(lowest);
}
/**
