This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 57319b452b9 SOLR-17951: Optimize re-ranking based on "functions"
57319b452b9 is described below
commit 57319b452b9bc03567b49884f91aa9ceaf66dbd9
Author: Chris Hostetter <[email protected]>
AuthorDate: Wed Apr 29 11:21:06 2026 -0700
SOLR-17951: Optimize re-ranking based on "functions"
(cherry picked from commit 930e6c158e9540b16dd0c8fb84972fe6fd9a5abf)
---
.../SOLR-17951-optimize-function-rerank.yml | 7 ++
.../apache/solr/search/AbstractReRankQuery.java | 21 +++++
.../org/apache/solr/search/ReRankOperator.java | 20 ++++-
.../apache/solr/search/ReRankQParserPlugin.java | 68 +++++++++++----
.../java/org/apache/solr/search/ReRankScaler.java | 19 ++---
.../solr/search/TestReRankQParserPlugin.java | 99 +++++++++++++++++++---
6 files changed, 190 insertions(+), 44 deletions(-)
diff --git a/changelog/unreleased/SOLR-17951-optimize-function-rerank.yml
b/changelog/unreleased/SOLR-17951-optimize-function-rerank.yml
new file mode 100644
index 00000000000..6331428370f
--- /dev/null
+++ b/changelog/unreleased/SOLR-17951-optimize-function-rerank.yml
@@ -0,0 +1,7 @@
+title: Optimize re-ranking based on "functions"
+type: added
+authors:
+- name: hossman
+links:
+- name: SOLR-17951
+ url: https://issues.apache.org/jira/browse/SOLR-17951
diff --git a/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java
b/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java
index a786e849108..5c026e55762 100644
--- a/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java
+++ b/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.search;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
@@ -59,6 +60,26 @@ public abstract class AbstractReRankQuery extends RankQuery {
this.reRankQueryRescorer = reRankQueryRescorer;
}
+ @VisibleForTesting
+ int getReRankDocs() {
+ return reRankDocs;
+ }
+
+ @VisibleForTesting
+ Rescorer getRescorer() {
+ return reRankQueryRescorer;
+ }
+
+ @VisibleForTesting
+ ReRankOperator getReRankOperator() {
+ return reRankOperator;
+ }
+
+ @VisibleForTesting
+ ReRankScaler getReRankScaler() {
+ return reRankScaler;
+ }
+
@Override
public RankQuery wrap(Query _mainQuery) {
if (_mainQuery != null) {
diff --git a/solr/core/src/java/org/apache/solr/search/ReRankOperator.java
b/solr/core/src/java/org/apache/solr/search/ReRankOperator.java
index 0b6583e095e..b3b5582836c 100644
--- a/solr/core/src/java/org/apache/solr/search/ReRankOperator.java
+++ b/solr/core/src/java/org/apache/solr/search/ReRankOperator.java
@@ -17,12 +17,24 @@
package org.apache.solr.search;
import java.util.Locale;
+import java.util.function.DoubleBinaryOperator;
import org.apache.solr.common.SolrException;
-public enum ReRankOperator {
- ADD,
- MULTIPLY,
- REPLACE;
+public enum ReRankOperator implements DoubleBinaryOperator {
+ ADD((firstPass, secondPass) -> firstPass + secondPass),
+ MULTIPLY((firstPass, secondPass) -> firstPass * secondPass),
+ REPLACE((firstPass, secondPass) -> secondPass);
+
+ private final DoubleBinaryOperator op;
+
+ private ReRankOperator(final DoubleBinaryOperator op) {
+ this.op = op;
+ }
+
+ @Override
+ public double applyAsDouble(final double firstPass, final double secondPass)
{
+ return op.applyAsDouble(firstPass, secondPass);
+ }
public static ReRankOperator get(String p) {
if (p != null) {
diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java
b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java
index 53373c03de5..33bb7f2a35b 100644
--- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java
@@ -18,9 +18,15 @@ package org.apache.solr.search;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.function.DoubleBinaryOperator;
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.DoubleValuesSourceRescorer;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryRescorer;
+import org.apache.lucene.search.Rescorer;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
@@ -63,6 +69,26 @@ public class ReRankQParserPlugin extends QParserPlugin {
return new ReRankQParser(query, localParams, params, req);
}
+ /**
+ * Helper method for constructing a {@link Rescorer} from a {@link
#RERANK_QUERY}, {@link
+ * #RERANK_WEIGHT}, and {@link #RERANK_OPERATOR}.
+ *
+ * <p>By default, this returns a customized {@link QueryRescorer}, unless
the {@link
+ * #RERANK_QUERY} is a known type that can more efficiently be re-ranked
using a customized {@link
+ * DoubleValuesSourceRescorer}.
+ */
+ private static Rescorer createRescorer(
+ final Query reRankQuery, final double reRankWeight, final ReRankOperator
reRankOperator) {
+ assert null != reRankQuery;
+ return switch (reRankQuery) {
+ case FunctionQuery functionQuery -> new ReRankDoubleValuesSourceRescorer(
+ functionQuery.getValueSource().asDoubleValuesSource(), reRankWeight,
reRankOperator);
+ case FunctionScoreQuery functionQuery -> new
ReRankDoubleValuesSourceRescorer(
+ functionQuery.getSource(), reRankWeight, reRankOperator);
+ default -> new ReRankQueryRescorer(reRankQuery, reRankWeight,
reRankOperator);
+ };
+ }
+
private static class ReRankQParser extends QParser {
private boolean isExplainResults() {
@@ -135,7 +161,7 @@ public class ReRankQParserPlugin extends QParserPlugin {
reRankScale,
reRankScaleWeight,
reRankOperator,
- new ReRankQueryRescorer(reRankQuery, 1, ReRankOperator.REPLACE),
+ createRescorer(reRankQuery, 1, ReRankOperator.REPLACE),
explainResults);
if (reRankScaler.scaleScores()) {
@@ -148,6 +174,28 @@ public class ReRankQParserPlugin extends QParserPlugin {
}
}
+ private static final class ReRankDoubleValuesSourceRescorer extends
DoubleValuesSourceRescorer {
+ final DoubleBinaryOperator scoreCombiner;
+
+ public ReRankDoubleValuesSourceRescorer(
+ final DoubleValuesSource valuesSource,
+ final double reRankWeight,
+ final ReRankOperator reRankOperator) {
+ super(valuesSource);
+ this.scoreCombiner =
+ (score, value) -> reRankOperator.applyAsDouble(score, reRankWeight *
value);
+ }
+
+ @Override
+ protected float combine(
+ final float firstPassScore, final boolean valuePresent, final double
sourceValue) {
+ if (valuePresent) {
+ return (float) scoreCombiner.applyAsDouble(firstPassScore,
sourceValue);
+ }
+ return firstPassScore;
+ }
+ }
+
private static final class ReRankQueryRescorer extends QueryRescorer {
final BiFloatFunction scoreCombiner;
@@ -160,20 +208,8 @@ public class ReRankQParserPlugin extends QParserPlugin {
public ReRankQueryRescorer(
Query reRankQuery, double reRankWeight, ReRankOperator reRankOperator)
{
super(reRankQuery);
- switch (reRankOperator) {
- case ADD:
- scoreCombiner = (score, second) -> (float) (score + reRankWeight *
second);
- break;
- case MULTIPLY:
- scoreCombiner = (score, second) -> (float) (score * reRankWeight *
second);
- break;
- case REPLACE:
- scoreCombiner = (score, second) -> (float) (reRankWeight * second);
- break;
- default:
- scoreCombiner = null;
- throw new IllegalArgumentException("Unexpected: reRankOperator=" +
reRankOperator);
- }
+ scoreCombiner =
+ (score, second) -> (float) reRankOperator.applyAsDouble(score,
reRankWeight * second);
}
@Override
@@ -226,7 +262,7 @@ public class ReRankQParserPlugin extends QParserPlugin {
super(
defaultQuery,
reRankDocs,
- new ReRankQueryRescorer(reRankQuery, reRankWeight, reRankOperator),
+ createRescorer(reRankQuery, reRankWeight, reRankOperator),
reRankScaler,
reRankOperator);
this.reRankQuery = reRankQuery;
diff --git a/solr/core/src/java/org/apache/solr/search/ReRankScaler.java
b/solr/core/src/java/org/apache/solr/search/ReRankScaler.java
index 3a03e13fc65..1cf34fffeae 100644
--- a/solr/core/src/java/org/apache/solr/search/ReRankScaler.java
+++ b/solr/core/src/java/org/apache/solr/search/ReRankScaler.java
@@ -23,7 +23,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.QueryRescorer;
+import org.apache.lucene.search.Rescorer;
import org.apache.lucene.search.ScoreDoc;
public class ReRankScaler {
@@ -35,7 +35,7 @@ public class ReRankScaler {
protected boolean explainResults;
protected ReRankOperator reRankOperator;
protected ReRankScalerExplain reRankScalerExplain;
- private QueryRescorer replaceRescorer;
+ private Rescorer replaceRescorer;
private Set<Integer> reRankSet;
private double reRankScaleWeight;
@@ -44,7 +44,7 @@ public class ReRankScaler {
String reRankScale,
double reRankScaleWeight,
ReRankOperator reRankOperator,
- QueryRescorer replaceRescorer,
+ Rescorer replaceRescorer,
boolean explainResults)
throws SyntaxError {
@@ -99,7 +99,7 @@ public class ReRankScaler {
}
}
- public QueryRescorer getReplaceRescorer() {
+ public Rescorer getReplaceRescorer() {
return replaceRescorer;
}
@@ -237,16 +237,7 @@ public class ReRankScaler {
float reRankScore,
double reRankScaleWeight,
ReRankOperator reRankOperator) {
- switch (reRankOperator) {
- case ADD:
- return (float) (orginalScore + reRankScaleWeight * reRankScore);
- case REPLACE:
- return (float) (reRankScaleWeight * reRankScore);
- case MULTIPLY:
- return (float) (orginalScore * reRankScaleWeight * reRankScore);
- default:
- return -1;
- }
+ return (float) reRankOperator.applyAsDouble(orginalScore,
reRankScaleWeight * reRankScore);
}
public static final class ReRankScalerExplain {
diff --git
a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
index e6856a19e93..f3149af82bc 100644
--- a/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestReRankQParserPlugin.java
@@ -16,16 +16,24 @@
*/
package org.apache.solr.search;
+import static org.hamcrest.CoreMatchers.instanceOf;
+
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.apache.lucene.search.DoubleValuesSourceRescorer;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryRescorer;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.SolrMetricTestUtils;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -63,6 +71,66 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4 {
assertEquals(ReRankQParserPlugin.RERANK_OPERATOR, "reRankOperator");
}
+ public void testIntrospection() throws Exception {
+ final SolrQueryResponse rsp = new SolrQueryResponse();
+ try (SolrQueryRequest req = req(params("r_f", "{!func}field(test_ti)",
"r_q", "id:1^=10"))) {
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+
+ { // Sanity check defaults w/simple rank query
+ final AbstractReRankQuery q = parseAndCast("{!rerank
reRankQuery=$r_q}", req);
+ assertEquals(ReRankQParserPlugin.RERANK_DOCS_DEFAULT,
q.getReRankDocs());
+ assertEquals(ReRankOperator.ADD, q.getReRankOperator());
+ assertThat(q.getRescorer(), instanceOf(QueryRescorer.class));
+ assertFalse(q.getReRankScaler().scaleScores());
+ }
+
+ { // Check defaults with function rank query (using an optimized value
source based rescorer)
+ final AbstractReRankQuery q = parseAndCast("{!rerank
reRankQuery=$r_f}", req);
+ assertEquals(ReRankQParserPlugin.RERANK_DOCS_DEFAULT,
q.getReRankDocs());
+ assertEquals(ReRankOperator.ADD, q.getReRankOperator());
+ assertThat(q.getRescorer(),
instanceOf(DoubleValuesSourceRescorer.class));
+ assertFalse(q.getReRankScaler().scaleScores());
+ }
+
+ { // check a re-ranker w/rescaling
+ final AbstractReRankQuery q =
+ parseAndCast(
+ "{!rerank reRankQuery=$r_q reRankOperator=replace
reRankScale='0-1'}", req);
+ assertEquals(ReRankQParserPlugin.RERANK_DOCS_DEFAULT,
q.getReRankDocs());
+ assertEquals(ReRankOperator.REPLACE, q.getReRankOperator());
+ assertThat(q.getRescorer(), instanceOf(QueryRescorer.class));
+ assertTrue(q.getReRankScaler().scaleScores());
+ assertTrue(q.getReRankScaler().scaleReRankScores());
+ assertFalse(q.getReRankScaler().scaleMainScores());
+ assertEquals(0, q.getReRankScaler().getReRankQueryMin());
+ assertEquals(1, q.getReRankScaler().getReRankQueryMax());
+ assertThat(q.getReRankScaler().getReplaceRescorer(),
instanceOf(QueryRescorer.class));
+ }
+
+ { // check a function re-ranker w/rescaling
+ final AbstractReRankQuery q =
+ parseAndCast(
+ "{!rerank reRankQuery=$r_f reRankOperator=multiply
reRankScale='1-2' reRankMainScale=0-3}",
+ req);
+ assertEquals(ReRankQParserPlugin.RERANK_DOCS_DEFAULT,
q.getReRankDocs());
+ assertEquals(ReRankOperator.MULTIPLY, q.getReRankOperator());
+ assertThat(q.getRescorer(),
instanceOf(DoubleValuesSourceRescorer.class));
+ assertTrue(q.getReRankScaler().scaleScores());
+ assertTrue(q.getReRankScaler().scaleReRankScores());
+ assertTrue(q.getReRankScaler().scaleMainScores());
+ assertEquals(1, q.getReRankScaler().getReRankQueryMin());
+ assertEquals(2, q.getReRankScaler().getReRankQueryMax());
+ assertEquals(0, q.getReRankScaler().getMainQueryMin());
+ assertEquals(3, q.getReRankScaler().getMainQueryMax());
+ assertThat(
+ q.getReRankScaler().getReplaceRescorer(),
instanceOf(DoubleValuesSourceRescorer.class));
+ }
+
+ } finally {
+ SolrRequestInfo.clearRequestInfo();
+ }
+ }
+
@Test
public void testRerankReturnOriginalScore() throws Exception {
@@ -115,7 +183,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4
{
+ ReRankQParserPlugin.RERANK_DOCS
+ "=200}");
params.add("q", "term_s:YYYY");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -189,7 +257,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4
{
+ ReRankQParserPlugin.RERANK_DOCS
+ "=200}");
params.add("q", "term_s:YYYY");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -252,7 +320,7 @@ public class TestReRankQParserPlugin extends SolrTestCaseJ4
{
+ ReRankQParserPlugin.RERANK_DOCS
+ "=200}");
params.add("q", "term_s:YYYY");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -290,7 +358,11 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
+ "=200}";
params.add("rq", rerankQueryByOp.apply(operation));
params.add("q", "term_s:YYYY^=0.1"); // force score=0.1
- params.add("rqq", "{!edismax bf=$bff}*:*"); // returns 1 + $bff
+ params.add(
+ "rqq",
+ random().nextBoolean()
+ ? "{!edismax bf=$bff}*:*"
+ : "{!func}sum(1.0,$bff)"); // returns 1 + $bff
params.add("bff", "field(test_ti)"); // test_ti=5000 for item 3
params.add("start", "0");
params.add("rows", "6");
@@ -1362,7 +1434,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
+ "=200}");
params.add("q", "term_t:YYYY");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -1403,7 +1475,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
+ "=200}");
params.add("q", "term_t:YYYY");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "4");
@@ -1482,7 +1554,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
+ "=4}");
params.add("q", "term_t:YYYY");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -1523,7 +1595,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
params.add("q", "term_t:YYYY");
params.add("fq", "id:(4 OR 5)");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -1556,7 +1628,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
params.add("q", "term_t:YYYY");
params.add("fq", "id:(4 OR 5)");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -1596,7 +1668,7 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
+ "=4}");
params.add("q", "term_t:YYYY");
params.add("fl", "id,score");
- params.add("rqq", "{!edismax bf=$bff}*:*");
+ params.add("rqq", random().nextBoolean() ? "{!edismax bf=$bff}*:*" :
"{!func}sum(1.0,$bff)");
params.add("bff", "field(test_ti)");
params.add("start", "0");
params.add("rows", "6");
@@ -1744,4 +1816,11 @@ public class TestReRankQParserPlugin extends
SolrTestCaseJ4 {
assertTrue(explainResponse.contains("10.0 = scaled main query score
between: 10-20"));
}
+
+ private static AbstractReRankQuery parseAndCast(final String query, final
SolrQueryRequest req)
+ throws Exception {
+ final Query q = QParser.getParser(query, req).getQuery();
+ assertThat(q, instanceOf(AbstractReRankQuery.class));
+ return (AbstractReRankQuery) q;
+ }
}
