This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 163df6a35b6 SOLR-18335 : SPLITSHARD now supports router.field with
docValues (#4717)
163df6a35b6 is described below
commit 163df6a35b60c3b2ead5a43e599f51420191edc2
Author: Olivier Boudet <[email protected]>
AuthorDate: Tue Sep 8 22:41:47 2026 +0200
SOLR-18335 : SPLITSHARD now supports router.field with docValues (#4717)
SPLITSHARD has expanded support for collection router.field, not only
supporting indexed fields but also docValues.
Note: "Point" numeric fields only work in this case with docValues.
Co-authored-by: David Smiley <[email protected]>
---
...R-18335-splitshard-router-field-point-field.yml | 10 ++
.../org/apache/solr/update/SolrIndexSplitter.java | 154 +++++++++++++++++----
.../solr/cloud/api/collections/ShardSplitTest.java | 70 ++++++++++
.../apache/solr/update/SolrIndexSplitterTest.java | 91 ++++++++++++
.../pages/collection-management.adoc | 5 +-
5 files changed, 303 insertions(+), 27 deletions(-)
diff --git
a/changelog/unreleased/SOLR-18335-splitshard-router-field-point-field.yml
b/changelog/unreleased/SOLR-18335-splitshard-router-field-point-field.yml
new file mode 100644
index 00000000000..dab08ed0639
--- /dev/null
+++ b/changelog/unreleased/SOLR-18335-splitshard-router-field-point-field.yml
@@ -0,0 +1,10 @@
+title: >
+ SPLITSHARD has expanded support for collection router.field, not only
supporting indexed fields but also docValues.
+ Note: "Point" numeric fields only work in this case with docValues.
+type: added
+authors:
+ - name: Olivier Boudet
+ - name: David Smiley
+links:
+ - name: SOLR-18335
+ url: https://issues.apache.org/jira/browse/SOLR-18335
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index 9312b831e75..b9e4c4722d1 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -41,6 +41,8 @@ import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.misc.store.HardlinkCopyDirectoryWrapper;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
@@ -679,7 +681,21 @@ public class SolrIndexSplitter {
Terms terms = reader.terms(field.getName());
TermsEnum termsEnum = terms == null ? null : terms.iterator();
- if (termsEnum == null) return docSets;
+ if (termsEnum == null) {
+ // No term dictionary for this field (e.g. a PointField, which has no
Terms; or a
+ // docValues-only field that isn't indexed). Derive route values from
docValues instead.
+ return splitUsingDocValues(
+ readerContext,
+ numPieces,
+ field,
+ rangesArr,
+ splitKey,
+ hashRouter,
+ delete,
+ docSets,
+ liveDocs,
+ currentPartition);
+ }
BytesRef term = null;
PostingsEnum postingsEnum = null;
@@ -746,38 +762,124 @@ public class SolrIndexSplitter {
}
if (docsMatchingRanges != null) {
- for (int ii = 0; ii < docsMatchingRanges.length; ii++) {
- if (0 == docsMatchingRanges[ii]) continue;
- switch (ii) {
- case 0:
- // document loss
- log.error(
- "Splitting {}: {} documents belong to no shards and will be
dropped",
- reader,
- docsMatchingRanges[ii]);
- break;
- case 1:
- // normal case, each document moves to one of the sub-shards
- log.info(
- "Splitting {}: {} documents will move into a sub-shard",
- reader,
- docsMatchingRanges[ii]);
- break;
- default:
- // document duplication
- log.error(
- "Splitting {}: {} documents will be moved to multiple ({})
sub-shards",
- reader,
- docsMatchingRanges[ii],
- ii);
- break;
+ logDocsMatchingRanges(reader, docsMatchingRanges);
+ }
+
+ return docSets;
+ }
+
+ private static FixedBitSet[] splitUsingDocValues(
+ LeafReaderContext readerContext,
+ int numPieces,
+ SchemaField field,
+ DocRouter.Range[] rangesArr,
+ String splitKey,
+ HashBasedRouter hashRouter,
+ boolean delete,
+ FixedBitSet[] docSets,
+ Bits liveDocs,
+ AtomicInteger currentPartition)
+ throws IOException {
+ if (!field.hasDocValues()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Route field '"
+ + field.getName()
+ + "' has no term index to split by and lacks docValues; unable
to split shard.");
+ }
+ LeafReader reader = readerContext.reader();
+ ValueSource valueSource = field.getType().getValueSource(field, null);
+ FunctionValues routeFieldValues = valueSource.getValues(Map.of(),
readerContext);
+
+ int[] docsMatchingRanges = null;
+ if (rangesArr != null) {
+ docsMatchingRanges = new int[rangesArr.length + 1];
+ }
+
+ for (int doc = 0; doc < reader.maxDoc(); doc++) {
+ if (liveDocs != null && !liveDocs.get(doc)) {
+ continue;
+ }
+
+ String routeValue = getRouteFieldValue(routeFieldValues, doc, field);
+ if (splitKey != null) {
+ String part1 = ((CompositeIdRouter)
hashRouter).getRouteKeyNoSuffix(routeValue);
+ if (!splitKey.equals(part1)) {
+ continue;
}
}
+
+ if (rangesArr == null) {
+ if (delete) {
+ docSets[currentPartition.get()].clear(doc);
+ } else {
+ docSets[currentPartition.get()].set(doc);
+ }
+ currentPartition.set((currentPartition.get() + 1) % numPieces);
+ } else {
+ int hash = hashRouter.sliceHash(routeValue, null, null, null);
+ int matchingRangesCount = 0;
+ for (int i = 0; i < rangesArr.length; i++) {
+ if (rangesArr[i].includes(hash)) {
+ if (delete) {
+ docSets[i].clear(doc);
+ } else {
+ docSets[i].set(doc);
+ }
+ ++matchingRangesCount;
+ }
+ }
+ docsMatchingRanges[matchingRangesCount]++;
+ }
}
+ if (docsMatchingRanges != null) {
+ logDocsMatchingRanges(reader, docsMatchingRanges);
+ }
return docSets;
}
+ private static String getRouteFieldValue(
+ FunctionValues routeFieldValues, int doc, SchemaField field) throws
IOException {
+ if (routeFieldValues.exists(doc)) {
+ return routeFieldValues.strVal(doc);
+ }
+
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to read route field '" + field.getName() + "' for shard
splitting.");
+ }
+
+ private static void logDocsMatchingRanges(LeafReader reader, int[]
docsMatchingRanges) {
+ for (int ii = 0; ii < docsMatchingRanges.length; ii++) {
+ if (0 == docsMatchingRanges[ii]) continue;
+ switch (ii) {
+ case 0:
+ // document loss
+ log.error(
+ "Splitting {}: {} documents belong to no shards and will be
dropped",
+ reader,
+ docsMatchingRanges[ii]);
+ break;
+ case 1:
+ // normal case, each document moves to one of the sub-shards
+ log.info(
+ "Splitting {}: {} documents will move into a sub-shard",
+ reader,
+ docsMatchingRanges[ii]);
+ break;
+ default:
+ // document duplication
+ log.error(
+ "Splitting {}: {} documents will be moved to multiple ({})
sub-shards",
+ reader,
+ docsMatchingRanges[ii],
+ ii);
+ break;
+ }
+ }
+ }
+
private static void checkRouterSupportsSplitKey(HashBasedRouter hashRouter,
String splitKey) {
if (splitKey != null && !(hashRouter instanceof CompositeIdRouter)) {
throw new IllegalStateException(
diff --git
a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
index 088eb242124..074d0431342 100644
---
a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
+++
b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
@@ -107,6 +107,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
incompleteOrOverlappingCustomRangeTest();
splitByUniqueKeyTest();
splitByRouteFieldTest();
+ splitByNumericRouteFieldTest();
splitByRouteKeyTest();
// todo can't call waitForThingsToLevelOut because it looks for jettys of
all shards
@@ -1009,6 +1010,75 @@ public class ShardSplitTest extends
BasicDistributedZkTest {
.query(new SolrQuery("*:*").setParam("shards", "shard1_1"))
.getResults()
.getNumFound());
+ assertEquals(101, collectionClient.query(new
SolrQuery("*:*")).getResults().getNumFound());
+ }
+ }
+
+ public void splitByNumericRouteFieldTest() throws Exception {
+ log.info("Starting splitByNumericRouteFieldTest");
+ String collectionName = "numericRouteFieldColl";
+ int numShards = 4;
+ int replicationFactor = 2;
+
+ HashMap<String, List<Integer>> collectionInfos = new HashMap<>();
+ String shardField = "shard_l";
+ try (CloudSolrClient client = createCloudClient(null)) {
+ Map<String, Object> props =
+ Map.of(
+ REPLICATION_FACTOR,
+ replicationFactor,
+ CollectionHandlingUtils.NUM_SLICES,
+ numShards,
+ "router.field",
+ shardField);
+
+ createCollection(collectionInfos, collectionName, props, client);
+ }
+
+ List<Integer> list = collectionInfos.get(collectionName);
+ checkForCollection(collectionName, list);
+
+ waitForRecoveriesToFinish(false);
+
+ getCommonCloudSolrClient();
+ String baseUrl = getBaseUrlFromZk(cloudClient.getClusterState(),
collectionName);
+
+ try (SolrClient collectionClient = getHttpSolrClient(baseUrl,
collectionName)) {
+ ClusterState clusterState = cloudClient.getClusterState();
+ final DocRouter router =
clusterState.getCollection(collectionName).getRouter();
+ Slice shard1 =
clusterState.getCollection(collectionName).getSlice(SHARD1);
+ DocRouter.Range shard1Range =
+ shard1.getRange() != null ? shard1.getRange() : router.fullRange();
+ final List<DocRouter.Range> ranges = router.partitionRange(2,
shard1Range);
+ final int[] docCounts = new int[ranges.size()];
+
+ for (int i = 100; i <= 200; i++) {
+ collectionClient.add(getDoc(id, i, "n_ti", i, shardField, i));
+ int idx = getHashRangeIdx(router, ranges, Integer.toString(i));
+ if (idx != -1) {
+ docCounts[idx]++;
+ }
+ }
+
+ collectionClient.commit();
+
+ trySplit(collectionName, null, SHARD1, 3);
+
+ waitForRecoveriesToFinish(collectionName, false);
+
+ assertEquals(
+ docCounts[0],
+ collectionClient
+ .query(new SolrQuery("*:*").setParam("shards", "shard1_0"))
+ .getResults()
+ .getNumFound());
+ assertEquals(
+ docCounts[1],
+ collectionClient
+ .query(new SolrQuery("*:*").setParam("shards", "shard1_1"))
+ .getResults()
+ .getNumFound());
+ assertEquals(101, collectionClient.query(new
SolrQuery("*:*")).getResults().getNumFound());
}
}
diff --git
a/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
index d7f7fa4f2e1..9ae688f377a 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
@@ -55,6 +55,10 @@ public class SolrIndexSplitterTest extends SolrTestCaseJ4 {
// _version_
System.setProperty("solr.directoryFactory",
"solr.NRTCachingDirectoryFactory");
System.setProperty("solr.tests.lockType",
DirectoryFactory.LOCK_TYPE_SIMPLE);
+ // route_l needs docValues to be usable as a router.field when numerics
are Point-based
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) {
+ System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true");
+ }
initCore("solrconfig.xml", "schema15.xml");
}
@@ -413,6 +417,16 @@ public class SolrIndexSplitterTest extends SolrTestCaseJ4 {
doTestSplitByRouteKey(SolrIndexSplitter.SplitMethod.LINK);
}
+ @Test
+ public void testSplitByNumericRouteField() throws Exception {
+ doTestSplitByNumericRouteField(SolrIndexSplitter.SplitMethod.REWRITE);
+ }
+
+ @Test
+ public void testSplitByNumericRouteFieldLink() throws Exception {
+ doTestSplitByNumericRouteField(SolrIndexSplitter.SplitMethod.LINK);
+ }
+
private void doTestSplitByRouteKey(SolrIndexSplitter.SplitMethod
splitMethod) throws Exception {
Path indexDir = createTempDir();
@@ -480,6 +494,83 @@ public class SolrIndexSplitterTest extends SolrTestCaseJ4 {
}
}
+ private void doTestSplitByNumericRouteField(SolrIndexSplitter.SplitMethod
splitMethod)
+ throws Exception {
+ CompositeIdRouter router = new CompositeIdRouter();
+ List<DocRouter.Range> ranges = router.partitionRange(2,
router.fullRange());
+ int[] expectedDocCounts = new int[ranges.size()];
+
+ for (int i = 100; i < 140; i++) {
+ String routeValue = Integer.toString(i);
+ assertU(adoc("id", "doc-" + i, "route_l", routeValue));
+
+ int hash = router.sliceHash(routeValue, null, null, null);
+ for (int rangeIndex = 0; rangeIndex < ranges.size(); rangeIndex++) {
+ if (ranges.get(rangeIndex).includes(hash)) {
+ expectedDocCounts[rangeIndex]++;
+ break;
+ }
+ }
+ }
+
+ assertU(commit());
+ assertJQ(req("q", "*:*"), "/response/numFound==40");
+
+ SolrQueryRequestBase request = null;
+ Directory directory = null;
+ try {
+ request = lrf.makeRequest("q", "dummy");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SplitIndexCommand command =
+ new SplitIndexCommand(
+ request,
+ rsp,
+ List.of(indexDir1.toString(), indexDir2.toString()),
+ null,
+ ranges,
+ router,
+ "route_l",
+ null,
+ splitMethod);
+ doSplit(command);
+
+ directory =
+ h.getCore()
+ .getDirectoryFactory()
+ .get(
+ indexDir1.toString(),
+ DirectoryFactory.DirContext.DEFAULT,
+ h.getCore().getSolrConfig().indexConfig.lockType);
+ DirectoryReader reader = DirectoryReader.open(directory);
+ assertEquals(
+ "split index1 has wrong number of documents", expectedDocCounts[0],
reader.numDocs());
+ reader.close();
+ h.getCore().getDirectoryFactory().release(directory);
+ directory = null;
+
+ directory =
+ h.getCore()
+ .getDirectoryFactory()
+ .get(
+ indexDir2.toString(),
+ DirectoryFactory.DirContext.DEFAULT,
+ h.getCore().getSolrConfig().indexConfig.lockType);
+ reader = DirectoryReader.open(directory);
+ assertEquals(
+ "split index2 has wrong number of documents", expectedDocCounts[1],
reader.numDocs());
+ reader.close();
+ h.getCore().getDirectoryFactory().release(directory);
+ directory = null;
+ } finally {
+ if (request != null) {
+ request.close();
+ }
+ if (directory != null) {
+ h.getCore().getDirectoryFactory().release(directory);
+ }
+ }
+ }
+
@Test
public void testSplitWithChildDocs() throws Exception {
doTestSplitWithChildDocs(SolrIndexSplitter.SplitMethod.REWRITE);
diff --git
a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
index 46452619f7c..45b26e0af47 100644
---
a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
+++
b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
@@ -232,6 +232,8 @@ For nested documents, the route field must match among all
the documents in the
+
Please note that xref:configuration-guide:realtime-get.adoc[] or retrieval by
document ID would also require the parameter `\_route_` (or `shard.keys`) to
avoid a distributed search.
+Use of docValues is highly encouraged on this field.
+
`perReplicaState`::
+
[%autowidth,frame=none]
@@ -1647,7 +1649,8 @@ s|Required |Default: none
What to name the backup that is created.
Provided as a query parameter for v1 requests, or as a path segment for v2
requests.
+
-For incremental backups, the backup name should be reused to add new backup
points to the existing backup. For non-incremental backups (deprecated), this
name is checked to ensure it doesn't already exist, and an error message is
raised if it does.
+For incremental backups, the backup name should be reused to add new backup
points to the existing backup.
+For non-incremental backups (deprecated), this name is checked to ensure it
doesn't already exist, and an error message is raised if it does.
`location`::
+
