This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 997a39ea30df63fd5af0917990b079b1fae134ad Author: Zhenyu Li <[email protected]> AuthorDate: Sun Sep 6 00:14:34 2026 -0400 SOLR-18413: Handle deleted MIGRATE routing targets (#4862) Problem fixed: Say the target of a MIGRATE collection is deleted, yet the source collection exists and receives an update request. The request reports failure after partially applying the mutation, leaving replicas of the source shard inconsistent. No update request is sent to the source replica. Therefore, SolrCmdDistributor does not observe a replica failure. Consequently, this request does not lower the replica's shard term or directly trigger recovery. (cherry picked from commit 465859177153921cd0c09fb9f6c4d643d2f30d41) --- .../fix-dangling-routing-rule-target.yml | 8 ++ .../processor/DistributedZkUpdateProcessor.java | 95 ++++++++++++---------- .../org/apache/solr/cloud/MigrateRouteKeyTest.java | 53 ++++++++++++ 3 files changed, 115 insertions(+), 41 deletions(-) diff --git a/changelog/unreleased/fix-dangling-routing-rule-target.yml b/changelog/unreleased/fix-dangling-routing-rule-target.yml new file mode 100644 index 00000000000..a3ba20b78bc --- /dev/null +++ b/changelog/unreleased/fix-dangling-routing-rule-target.yml @@ -0,0 +1,8 @@ +title: Prevent partial writes when a collection referenced by a MIGRATE routing rule has been deleted +type: fixed +authors: + - name: ZhenyuLi + nick: JHSUYU +links: + - name: SOLR-18413 + url: https://issues.apache.org/jira/browse/SOLR-18413 diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 35d96a7c14a..39724447530 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -1007,7 +1007,17 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor { int hash = compositeIdRouter.sliceHash(id, doc, null, coll); for (DocRouter.Range range : ranges) { if (range.includes(hash)) { - DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName()); + DocCollection targetColl = + cstate.getCollectionOrNull(rule.getTargetCollectionName()); + if (targetColl == null) { + if (log.isInfoEnabled()) { + log.info( + "Removing shard update routing rule because the target collection {} doesn't exist", + rule.getTargetCollectionName()); + } + removeRoutingRule(myShardId, routeKey); + break; + } Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl); if (activeSlices == null || activeSlices.isEmpty()) { @@ -1027,46 +1037,8 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor { } } } else { - ReentrantLock ruleExpiryLock = req.getCore().getRuleExpiryLock(); - if (!ruleExpiryLock.isLocked()) { - try { - if (ruleExpiryLock.tryLock(10, TimeUnit.MILLISECONDS)) { - log.info("Going to expire routing rule"); - try { - Map<String, Object> map = - Map.of( - Overseer.QUEUE_OPERATION, - OverseerAction.REMOVEROUTINGRULE.toLower(), - ZkStateReader.COLLECTION_PROP, - collection, - ZkStateReader.SHARD_ID_PROP, - myShardId, - "routeKey", - routeKey + "!"); - if (distributedClusterStateUpdater.isDistributedStateUpdate()) { - ZkNodeProps message = new ZkNodeProps(map); - distributedClusterStateUpdater.doSingleStateUpdate( - DistributedClusterStateUpdater.MutatingCommand.SliceRemoveRoutingRule, - message, - zkController.getOverseer().getSolrCloudManager(), - zkController.getOverseer().getZkStateReader()); - } else { - zkController.getOverseer().offerStateUpdate(Utils.toJSON(map)); - } - } catch (KeeperException e) { - log.warn( - "Exception while removing routing rule for route key: {}", routeKey, e); - } catch (Exception e) { - log.error( - "Exception while removing routing rule for route key: {}", routeKey, e); - } finally { - ruleExpiryLock.unlock(); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } + log.info("Removing shard update routing rule because it has expired"); + removeRoutingRule(myShardId, routeKey); } } } @@ -1075,6 +1047,47 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor { return nodes; } + private void removeRoutingRule(String shardId, String routeKey) { + ReentrantLock ruleExpiryLock = req.getCore().getRuleExpiryLock(); + if (ruleExpiryLock.isLocked()) { + return; + } + try { + if (ruleExpiryLock.tryLock(10, TimeUnit.MILLISECONDS)) { + try { + Map<String, Object> map = + Map.of( + Overseer.QUEUE_OPERATION, + OverseerAction.REMOVEROUTINGRULE.toLower(), + ZkStateReader.COLLECTION_PROP, + collection, + ZkStateReader.SHARD_ID_PROP, + shardId, + "routeKey", + routeKey + "!"); + if (distributedClusterStateUpdater.isDistributedStateUpdate()) { + ZkNodeProps message = new ZkNodeProps(map); + distributedClusterStateUpdater.doSingleStateUpdate( + DistributedClusterStateUpdater.MutatingCommand.SliceRemoveRoutingRule, + message, + zkController.getOverseer().getSolrCloudManager(), + zkController.getOverseer().getZkStateReader()); + } else { + zkController.getOverseer().offerStateUpdate(Utils.toJSON(map)); + } + } catch (KeeperException e) { + log.warn("Exception while removing routing rule for route key: {}", routeKey, e); + } catch (Exception e) { + log.error("Exception while removing routing rule for route key: {}", routeKey, e); + } finally { + ruleExpiryLock.unlock(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + private void doDefensiveChecks(DistribPhase phase, UpdateCommand updateCommand) { boolean isReplayOrPeersync = (updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0; diff --git a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java index f883adee31b..e2257b67257 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java @@ -100,6 +100,59 @@ public class MigrateRouteKeyTest extends SolrCloudTestCase { assertTrue(remoteSolrException.getMessage().contains("split.key cannot be null or empty")); } + @Test + public void updateSucceedsAfterMigrateTargetIsDeleted() throws Exception { + String sourceCollection = "deletedMigrateTarget-source"; + CollectionAdminRequest.createCollection(sourceCollection, "conf", 1, 2) + .process(cluster.getSolrClient()); + String targetCollection = "deletedMigrateTarget-target"; + CollectionAdminRequest.createCollection(targetCollection, "conf", 1, 1) + .process(cluster.getSolrClient()); + + cluster.getSolrClient().add(sourceCollection, new SolrInputDocument("id", "a!1")); + cluster.getSolrClient().commit(sourceCollection); + + invokeCollectionMigration( + CollectionAdminRequest.migrateData(sourceCollection, targetCollection, "a!") + .setForwardTimeout(45)); + waitForState( + "Expected to find routing rule for split key a", + sourceCollection, + c -> { + if (c == null) return false; + Map<String, RoutingRule> routingRules = c.getSlice("shard1").getRoutingRules(); + return routingRules != null && routingRules.containsKey("a!"); + }); + + CollectionAdminRequest.deleteCollection(targetCollection).process(cluster.getSolrClient()); + waitForState("Expected target collection deletion", targetCollection, c -> c == null); + + cluster.getSolrClient().add(sourceCollection, new SolrInputDocument("id", "a!2")); + cluster.getSolrClient().commit(sourceCollection); + + DocCollection sourceState = getCollectionState(sourceCollection); + assertEquals(2, sourceState.getSlice("shard1").getReplicas().size()); + for (Replica replica : sourceState.getSlice("shard1")) { + try (SolrClient replicaClient = getHttpSolrClient(replica)) { + SolrQuery query = new SolrQuery("id:\"a!2\""); + query.set("distrib", false); + assertEquals( + "Document missing from replica " + replica.getName(), + 1, + replicaClient.query(query).getResults().getNumFound()); + } + } + + waitForState( + "Expected dangling routing rule removal", + sourceCollection, + c -> { + if (c == null) return false; + Map<String, RoutingRule> routingRules = c.getSlice("shard1").getRoutingRules(); + return routingRules == null || !routingRules.containsKey("a!"); + }); + } + @Test public void multipleShardMigrateTest() throws Exception {
