This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 15893247cb685d1bcccbcd8ad2f8ac3b57afccaf Author: Puneet Ahuja <[email protected]> AuthorDate: Fri Oct 23 10:19:44 2026 +0530 SOLR-18550: Fix numFound and numFoundExact for CombinedQuery (#5172) For combined queries (e.g. RRF), numFound is the distinct fused document count when numFoundExact is true, and otherwise a lower bound of at least the largest sub-query's match count, marked inexact when a sub-query list was truncated by the fusion pool (shards.rows or start + rows). (cherry picked from commit 79b23df58daac14b92cc35f23b8b81f72098b4ba) (cherry picked from commit f7b4658e1632664b100b1ad515994ec2ca90fd36) Tests adapted to branch_9x's test style (no text blocks). --- .../SOLR-18550-combined-query-numfound.yml | 13 ++++++ .../handler/component/CombinedQueryComponent.java | 35 ++++++++++------ .../component/CombinedQueryComponentTest.java | 32 +++++++++++++-- .../DistributedCombinedQueryComponentTest.java | 47 ++++++++++++++++++++++ .../query-guide/pages/json-combined-query-dsl.adoc | 3 ++ 5 files changed, 114 insertions(+), 16 deletions(-) diff --git a/changelog/unreleased/SOLR-18550-combined-query-numfound.yml b/changelog/unreleased/SOLR-18550-combined-query-numfound.yml new file mode 100644 index 00000000000..702ef9458db --- /dev/null +++ b/changelog/unreleased/SOLR-18550-combined-query-numfound.yml @@ -0,0 +1,13 @@ +title: > + For combined queries, numFound is the distinct fused document count when numFoundExact + is true, and otherwise a lower bound of at least the largest sub-query's match count, + marked inexact when a sub-query list was truncated by the fusion pool (shards.rows + or start + rows). +type: fixed +authors: + - name: Puneet Ahuja + nick: punAhuja + - name: Justin Michael Coyne +links: + - name: SOLR-18550 + url: https://issues.apache.org/jira/browse/SOLR-18550 \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java index 6c76f314649..0eb9761ab60 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java @@ -314,8 +314,9 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo); } - long numFound = 0; boolean hitCountIsExact = true; + boolean someQueryListTruncated = false; + long maxQueryNumFound = 0; boolean thereArePartialResults = false; Boolean segmentTerminatedEarly = null; boolean maxHitsTerminatedEarly = false; @@ -348,7 +349,9 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw if (dl instanceof SolrDocumentList) { docs = Objects.requireNonNull((SolrDocumentList) dl); queryNumFound += docs.getNumFound(); - hitCountIsExact = hitCountIsExact && Boolean.FALSE.equals(docs.getNumFoundExact()); + if (hitCountIsExact && Boolean.FALSE.equals(docs.getNumFoundExact())) { + hitCountIsExact = false; + } } } failedShardCount += @@ -440,27 +443,26 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw shardDoc.scoreDependentFields = doc.getSubsetOfFields(scoreDependentFields); } shardDoc.sortFieldValues = unmarshalledSortFieldValues; - if (!queuePerQuery.push(shardDoc)) { - numFound--; - } + queuePerQuery.push(shardDoc); } // end for-each-doc-in-response } // end for-each-response List<ShardDoc> shardDocsPerQuery = new ArrayList<>(queuePerQuery.resultIds(0).values()); shardDocsPerQuery.sort(Comparator.comparingInt(a -> a.positionInResponse)); shardDocMap.put(queriesToCombineKeys[queryIndex], shardDocsPerQuery); - numFound = max(numFound, queryNumFound); + // a sub-query list is complete iff every match reached the fusion pool + someQueryListTruncated |= queryNumFound != shardDocsPerQuery.size(); + maxQueryNumFound = max(maxQueryNumFound, queryNumFound); approximateTotalHits = max(approximateTotalHits, queryApproximateTotalHits); } // for each query to combine - rb.rsp.addToLog("hits", numFound); - SolrDocumentList responseDocs = new SolrDocumentList(); - responseDocs.setNumFound(numFound); - responseDocs.setNumFoundExact(hitCountIsExact); + responseDocs.setNumFoundExact(hitCountIsExact && !someQueryListTruncated); responseDocs.setStart(ss.getOffset()); - rb.resultIds = computeResultIdsWithCombiner(rb, shardDocMap, responseDocs, sortFields); + rb.resultIds = + computeResultIdsWithCombiner(rb, shardDocMap, responseDocs, sortFields, maxQueryNumFound); rb.setResponseDocs(responseDocs); + rb.rsp.addToLog("hits", responseDocs.getNumFound()); populateNextCursorMarkFromMergedShards(rb); @@ -584,8 +586,10 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw * @param rb The ResponseBuilder containing the request and context, such as sort specifications. * @param shardDocMap A map from shard addresses to the list of documents returned by each shard. * @param responseDocs The final response document list, which will be populated with null - * placeholders and have its max score set. + * placeholders and have its numFound and max score set. * @param sortFields An array of field for sorting to be applied. + * @param maxQueryNumFound The largest sub-query's match count, summed across shards; a floor for + * the reported numFound. * @return A map from document IDs to the corresponding ShardDoc objects for the documents in the * final sorted page of results. */ @@ -593,13 +597,18 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw ResponseBuilder rb, Map<String, List<ShardDoc>> shardDocMap, SolrDocumentList responseDocs, - SortField[] sortFields) { + SortField[] sortFields, + long maxQueryNumFound) { String algorithm = rb.req.getParams().get(CombinerParams.COMBINER_ALGORITHM, CombinerParams.DEFAULT_COMBINER); QueryAndResponseCombiner combinerStrategy = QueryAndResponseCombiner.getImplementation(algorithm, combiners); List<ShardDoc> combinedShardDocs = combinerStrategy.combine(shardDocMap, rb.req.getParams()); + // the fused list is deduped by doc id, so its size is the distinct doc count; numFound is + // that count, but never below the largest sub-query's match count + responseDocs.setNumFound(max(maxQueryNumFound, combinedShardDocs.size())); + // adding explanation for the ordered shard docs as debug info if (rb.isDebugResults()) { String[] queryKeys = rb.req.getParams().getParams(CombinerParams.COMBINER_QUERY); diff --git a/solr/core/src/test/org/apache/solr/handler/component/CombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CombinedQueryComponentTest.java index 40ae4701e58..db13dfab424 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/CombinedQueryComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/CombinedQueryComponentTest.java @@ -125,18 +125,42 @@ public class CombinedQueryComponentTest extends BaseDistributedSearchTestCase { query( CommonParams.JSON, "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"title:title test for doc 1\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"text:test text for doc 2\"}}}," + + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\":{\"lucene\":{\"query\":\"id:(4^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + "\"limit\":5," + "\"fields\":[\"id\",\"score\",\"title\"]," + "\"params\":{\"combiner\":true,\"debug\":[\"results\"],\"combiner.query\":[\"lexical1\",\"lexical2\"]," + "\"rid\": \"test-1\"}}", CommonParams.QT, "/search"); - assertEquals(10, rsp.getResults().getNumFound()); + // both queries match 4 docs, overlapping on doc 5, so the fused result has 7 distinct docs + assertEquals(7, rsp.getResults().getNumFound()); assertTrue(rsp.getDebugMap().containsKey("combinerExplanations")); } + /** + * Tests that numFound is the distinct count of fused documents, not the larger sub-query's match + * count. Two queries matching three and two distinct docs, no overlap. + */ + @Test + public void testNumFoundIsFusedCount() throws Exception { + prepareIndexDocs(); + QueryResponse rsp = + query( + CommonParams.JSON, + "{\"queries\":" + + "{\"cats\":{\"lucene\":{\"query\":\"id:(1^=3 OR 2^=2 OR 3^=1)\"}}," + + "\"dogs\":{\"lucene\":{\"query\":\"id:(4^=2 OR 5^=1)\"}}}," + + "\"limit\":10," + + "\"fields\":[\"id\"]," + + "\"params\":{\"combiner\":true,\"combiner.query\":[\"cats\",\"dogs\"]}}", + CommonParams.QT, + "/search"); + assertEquals(5, rsp.getResults().size()); + assertEquals(5, rsp.getResults().getNumFound()); + assertEquals(Boolean.TRUE, rsp.getResults().getNumFoundExact()); + } + /** Test no results in combined queries. */ @Test public void testNoResults() throws Exception { @@ -196,6 +220,8 @@ public class CombinedQueryComponentTest extends BaseDistributedSearchTestCase { + ":[\"lexical1\",\"lexical2\"],\"debug\":[\"results\"]}}", CommonParams.QT, "/search"); + // the test combiner fuses to an empty list, so numFound falls back to the largest + // sub-query's match count rather than the (empty) fused pool assertEquals(10, rsp.getResults().getNumFound()); assertEquals( "org.apache.lucene.search.Explanation:30 = this is test combiner\n", diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java index 4303a1eb238..00570dd5268 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java @@ -21,6 +21,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import org.apache.solr.BaseDistributedSearchTestCase; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -162,6 +163,52 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3"); } + /** + * Tests numFound and numFoundExact for fused results, including a truncated fusion pool and a + * count-only request. + * + * @throws Exception if any error occurs during the test execution + */ + @Test + @ShardsFixed(num = 2) + public void testNumFound() throws Exception { + prepareIndexDocs(); + String queries = + "{\"queries\":" + + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(1^=6 OR 2^=5 OR 3^=4 OR 4^=3 OR 5^=2 OR 6^=1)\"}}," + + "\"lexical2\":{\"lucene\":{\"query\":\"id:(5^=6 OR 6^=5 OR 7^=4 OR 8^=3 OR 9^=2 OR 10^=1)\"}}}," + + "\"limit\":%d," + + "\"fields\":[\"id\",\"score\"]," + + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}"; + + // both queries match 6 docs, overlapping on docs 5 and 6, so the fused result has 10 distinct + // docs; numFound is that distinct count, exact since no list was truncated + QueryResponse rsp = + query( + CommonParams.JSON, String.format(Locale.ROOT, queries, 10), CommonParams.QT, "/search"); + assertEquals(10, rsp.getResults().getNumFound()); + assertEquals(Boolean.TRUE, rsp.getResults().getNumFoundExact()); + + // the fusion pool (start + rows = 2) truncates each 6-match list to its top 2; numFound never + // drops below the largest sub-query's match count (6) and is reported inexact because the + // fused pool holds only a subset of the matches + rsp = + query( + CommonParams.JSON, String.format(Locale.ROOT, queries, 2), CommonParams.QT, "/search"); + assertEquals(2, rsp.getResults().size()); + assertEquals(6, rsp.getResults().getNumFound()); + assertEquals(Boolean.FALSE, rsp.getResults().getNumFoundExact()); + + // count-only: rows=0 leaves the fusion pool empty, but numFound still reports the largest + // sub-query's match count rather than the pool size of 0 + rsp = + query( + CommonParams.JSON, String.format(Locale.ROOT, queries, 0), CommonParams.QT, "/search"); + assertEquals(0, rsp.getResults().size()); + assertEquals(6, rsp.getResults().getNumFound()); + assertEquals(Boolean.FALSE, rsp.getResults().getNumFoundExact()); + } + /** * Test multiple query execution with sort. * diff --git a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc index 61b7aed0b82..7af12e2681e 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc @@ -79,6 +79,9 @@ The query structure is similar to JSON Query DSL except for how multiple queries [NOTE] ==== Consider setting `shards.rows` to at least your deepest expected `start + rows` to enlarge the per-shard candidate pool used for fusion, giving more consistent ordering across pages. + +`numFound` in the response is the number of distinct documents in the fused result when `numFoundExact` is true. +Otherwise it is a lower bound — at least the largest sub-query's match count — because a sub-query matched more documents than were retained for fusion (`shards.rows` or `start + rows`). ==== === Example
