This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 29c96eb55db SOLR-17182: Part 1: add ExitableDirectoryReader benchmark.
(#3739)
29c96eb55db is described below
commit 29c96eb55db987e4091d73ac8cf26e92bbf1fc94
Author: Andrzej Białecki <[email protected]>
AuthorDate: Thu Oct 9 12:52:47 2025 +0200
SOLR-17182: Part 1: add ExitableDirectoryReader benchmark. (#3739)
Refactor CallerSpecificQueryLimit to create a reusable CallerMatcher.
---
solr/benchmark/build.gradle | 1 +
.../search/ExitableDirectoryReaderSearch.java | 197 +++++++++++++++
.../org/apache/solr/bench/search/JsonFaceting.java | 29 ++-
.../solr/core/ExitableDirectoryReaderTest.java | 9 +-
.../org/apache/solr/search/TestQueryLimits.java | 14 +-
.../solr/search/CallerSpecificQueryLimit.java | 178 ++------------
.../java/org/apache/solr/util/CallerMatcher.java | 270 +++++++++++++++++++++
.../solr/search/CallerSpecificQueryLimitTest.java | 6 +-
8 files changed, 524 insertions(+), 180 deletions(-)
diff --git a/solr/benchmark/build.gradle b/solr/benchmark/build.gradle
index bf32ee89457..48c97747f04 100644
--- a/solr/benchmark/build.gradle
+++ b/solr/benchmark/build.gradle
@@ -42,6 +42,7 @@ task echoCp {
dependencies {
implementation project(':solr:test-framework')
+ implementation project(':solr:core')
implementation project(':solr:solrj')
implementation project(':solr:solrj-streaming')
diff --git
a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java
b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java
new file mode 100644
index 00000000000..dc5e3af9f4d
--- /dev/null
+++
b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.bench.search;
+
+import static org.apache.solr.bench.BaseBenchState.log;
+import static org.apache.solr.bench.generators.SourceDSL.integers;
+import static org.apache.solr.bench.generators.SourceDSL.strings;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.bench.Docs;
+import org.apache.solr.bench.MiniClusterState;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.search.CallerSpecificQueryLimit;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.TestInjection;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(value = 1)
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(time = 20, iterations = 2)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Measurement(time = 30, iterations = 4)
+@Threads(value = 1)
+public class ExitableDirectoryReaderSearch {
+
+ static final String COLLECTION = "c1";
+
+ @State(Scope.Benchmark)
+ public static class BenchState {
+
+ Docs queryFields;
+
+ int NUM_DOCS = 500_000;
+ int WORDS = NUM_DOCS / 100;
+
+ @Setup(Level.Trial)
+ public void setupTrial(MiniClusterState.MiniClusterBenchState
miniClusterState)
+ throws Exception {
+ miniClusterState.setUseHttp1(true);
+ System.setProperty("documentCache.enabled", "false");
+ System.setProperty("queryResultCache.enabled", "false");
+ System.setProperty("filterCache.enabled", "false");
+ System.setProperty("miniClusterBaseDir", "build/work/mini-cluster");
+ // create a lot of small segments
+ System.setProperty("segmentsPerTier", "200");
+ System.setProperty("maxBufferedDocs", "100");
+
+ miniClusterState.startMiniCluster(1);
+ log("######### Creating index ...");
+ miniClusterState.createCollection(COLLECTION, 1, 1);
+ // create a lot of large-ish fields to scan positions
+ Docs docs =
+ Docs.docs(1234567890L)
+ .field("id", integers().incrementing())
+ .field("f1_ts",
strings().alpha().maxCardinality(WORDS).ofLengthBetween(3, 10))
+ .field(
+ "f2_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f3_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f4_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f5_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f6_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f7_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f8_ts",
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))
+ .field(
+ "f9_ts",
+
strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10));
+ miniClusterState.index(COLLECTION, docs, NUM_DOCS, true);
+ miniClusterState.forceMerge(COLLECTION, 200);
+ miniClusterState.dumpCoreInfo();
+ }
+
+ @Param({"false", "true"})
+ boolean useEDR;
+
+ // this adds significant processing time to the checking of query limits
+ // both to verify that it's actually used and to illustrate the impact of
limit checking
+ @Param({"false", "true"})
+ boolean verifyEDRInUse = true;
+
+ private static final String matchExpression = "ExitableTermsEnum:-1";
+
+ @Setup(Level.Iteration)
+ public void setupQueries(MiniClusterState.MiniClusterBenchState state)
throws Exception {
+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY,
String.valueOf(useEDR));
+ if (verifyEDRInUse) {
+ TestInjection.queryTimeout = new
CallerSpecificQueryLimit(Set.of(matchExpression));
+ }
+ // reload collection to force searcher / reader refresh
+ CollectionAdminRequest.Reload reload =
CollectionAdminRequest.reloadCollection(COLLECTION);
+ state.client.request(reload);
+
+ queryFields =
+ Docs.docs(1234567890L)
+ .field("id", integers().incrementing())
+ .field("f1_ts",
strings().alpha().maxCardinality(WORDS).ofLengthBetween(3, 10))
+ .field(
+ "f2_ts",
strings().alpha().maxCardinality(WORDS).multi(5).ofLengthBetween(3, 10));
+ }
+
+ @TearDown(Level.Iteration)
+ public void tearDownTrial() throws Exception {
+ if (useEDR && verifyEDRInUse) {
+ CallerSpecificQueryLimit queryLimit = (CallerSpecificQueryLimit)
TestInjection.queryTimeout;
+ if (queryLimit == null) {
+ throw new RuntimeException("Missing setup!");
+ }
+ Map<String, Integer> callCounts =
queryLimit.getCallerMatcher().getCallCounts();
+ log("######### Caller specific stats:");
+ log("Call counts: " + callCounts);
+ if (callCounts.get(matchExpression) == null) {
+ throw new RuntimeException("Missing call counts!");
+ }
+ if (callCounts.get(matchExpression).intValue() == 0) {
+ throw new RuntimeException("No call counts!");
+ }
+ }
+ }
+ }
+
+ private static ModifiableSolrParams createInitialParams() {
+ ModifiableSolrParams params =
+ MiniClusterState.params("rows", "100", "timeAllowed", "1000", "fl",
"*");
+ return params;
+ }
+
+ @Benchmark
+ public void testShortQuery(
+ MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh,
BenchState state)
+ throws Exception {
+ SolrInputDocument queryDoc = state.queryFields.inputDocument();
+ ModifiableSolrParams params = createInitialParams();
+ params.set("q", "f1_ts:" + queryDoc.getFieldValue("f1_ts").toString());
+ QueryRequest queryRequest = new QueryRequest(params);
+ QueryResponse rsp = queryRequest.process(miniClusterState.client,
COLLECTION);
+ bh.consume(rsp);
+ }
+
+ @Benchmark
+ public void testLongQuery(
+ MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh,
BenchState state)
+ throws Exception {
+ SolrInputDocument queryDoc = state.queryFields.inputDocument();
+ ModifiableSolrParams params = createInitialParams();
+ StringBuilder query = new StringBuilder();
+ for (int i = 2; i < 10; i++) {
+ if (query.length() > 0) {
+ query.append(" ");
+ }
+ String fld = "f" + i + "_ts";
+ query.append(fld + ":\"" + queryDoc.getFieldValue(fld) + "\"~20");
+ }
+ params.set("q", query.toString());
+ QueryRequest queryRequest = new QueryRequest(params);
+ QueryResponse rsp = queryRequest.process(miniClusterState.client,
COLLECTION);
+ bh.consume(rsp);
+ }
+}
diff --git
a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
index 1595de56d3b..d83477b56f7 100755
--- a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java
@@ -28,6 +28,7 @@ import org.apache.solr.bench.MiniClusterState;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.search.SolrIndexSearcher;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@@ -43,6 +44,7 @@ import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Timeout;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.Blackhole;
/** A benchmark to experiment with the performance of json faceting. */
@BenchmarkMode(Mode.Throughput)
@@ -71,6 +73,12 @@ public class JsonFaceting {
@Param("4")
int numShards;
+ @Param({"false", "true"})
+ boolean useTimeLimit;
+
+ @Param({"false", "true"})
+ boolean useExitableDirectoryReader;
+
// DV, // DocValues, collect into ordinal array
// UIF, // UnInvertedField, collect into ordinal array
// DVHASH, // DocValues, collect into hash
@@ -99,8 +107,13 @@ public class JsonFaceting {
BenchmarkParams benchmarkParams,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws Exception {
- System.setProperty("maxMergeAtOnce", "30");
- System.setProperty("segmentsPerTier", "30");
+ System.setProperty("maxMergeAtOnce", "50");
+ System.setProperty("segmentsPerTier", "50");
+ if (useExitableDirectoryReader) {
+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY, "true");
+ } else {
+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY,
"false");
+ }
miniClusterState.startMiniCluster(nodeCount);
@@ -158,6 +171,11 @@ public class JsonFaceting {
+ " , f8:{type:terms, field:'facet_s', limit:2, sort:'x desc',
facet:{x:'countvals(int4_i_dv)'} } "
+ '}');
+ if (useTimeLimit) {
+ // high enough to return all results, but still affecting the
performance
+ params.set("timeAllowed", "5000");
+ }
+
// MiniClusterState.log("params: " + params + "\n");
}
@@ -175,10 +193,11 @@ public class JsonFaceting {
@Benchmark
@Timeout(time = 500, timeUnit = TimeUnit.SECONDS)
- public Object jsonFacet(
+ public void jsonFacet(
MiniClusterState.MiniClusterBenchState miniClusterState,
BenchState state,
- BenchState.ThreadState threadState)
+ BenchState.ThreadState threadState,
+ Blackhole bh)
throws Exception {
final var url =
miniClusterState.nodes.get(threadState.random.nextInt(state.nodeCount));
QueryRequest queryRequest = new QueryRequest(state.params);
@@ -190,6 +209,6 @@ public class JsonFaceting {
// MiniClusterState.log("result: " + result);
- return result;
+ bh.consume(result);
}
}
diff --git
a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
index 86e4dcc51a0..61f24092280 100644
--- a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
+++ b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.core;
import java.util.Map;
+import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.search.CallerSpecificQueryLimit;
import org.apache.solr.search.SolrIndexSearcher;
@@ -69,11 +70,11 @@ public class ExitableDirectoryReaderTest extends
SolrTestCaseJ4 {
// create a limit that will not trip but will report calls to shouldExit()
// NOTE: we need to use the inner class name to capture the calls
String callerExpr = "ExitableTermsEnum:-1";
- CallerSpecificQueryLimit queryLimit = new
CallerSpecificQueryLimit(callerExpr);
+ CallerSpecificQueryLimit queryLimit = new
CallerSpecificQueryLimit(Set.of(callerExpr));
TestInjection.queryTimeout = queryLimit;
String q = "name:a*";
assertJQ(req("q", q), assertionString);
- Map<String, Integer> callCounts = queryLimit.getCallCounts();
+ Map<String, Integer> callCounts =
queryLimit.getCallerMatcher().getCallCounts();
if (withExitableDirectoryReader) {
assertTrue(
"there should be some calls from ExitableTermsEnum: " + callCounts,
@@ -86,7 +87,7 @@ public class ExitableDirectoryReaderTest extends
SolrTestCaseJ4 {
// check that the limits are tripped in ExitableDirectoryReader if it's in
use
int maxCount = random().nextInt(10) + 1;
callerExpr = "ExitableTermsEnum:" + maxCount;
- queryLimit = new CallerSpecificQueryLimit(callerExpr);
+ queryLimit = new CallerSpecificQueryLimit(Set.of(callerExpr));
TestInjection.queryTimeout = queryLimit;
// avoid using the cache
q = "name:b*";
@@ -95,7 +96,7 @@ public class ExitableDirectoryReaderTest extends
SolrTestCaseJ4 {
} else {
assertJQ(req("q", q), assertionString);
}
- callCounts = queryLimit.getCallCounts();
+ callCounts = queryLimit.getCallerMatcher().getCallCounts();
if (withExitableDirectoryReader) {
assertTrue(
"there should be at least " + maxCount + " calls from
ExitableTermsEnum: " + callCounts,
diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java
b/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java
index 690b05e3f21..dbee1511295 100644
--- a/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java
+++ b/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java
@@ -17,6 +17,7 @@
package org.apache.solr.search;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -79,7 +80,7 @@ public class TestQueryLimits extends SolrCloudTestCase {
"FacetComponent.process:2"
};
for (String matchingExpr : matchingExprTests) {
- CallerSpecificQueryLimit limit = new
CallerSpecificQueryLimit(matchingExpr);
+ CallerSpecificQueryLimit limit = new
CallerSpecificQueryLimit(Set.of(matchingExpr));
TestInjection.queryTimeout = limit;
rsp =
solrClient.query(
@@ -98,11 +99,14 @@ public class TestQueryLimits extends SolrCloudTestCase {
assertNotNull(
"should have partial results for expr " + matchingExpr,
rsp.getHeader().get("partialResults"));
- assertFalse("should have trippedBy info",
limit.getTrippedBy().isEmpty());
+ assertFalse("should have trippedBy info",
limit.getCallerMatcher().getTrippedBy().isEmpty());
assertTrue(
- "expected result to start with " + matchingExpr + " but was " +
limit.getTrippedBy(),
- limit.getTrippedBy().iterator().next().startsWith(matchingExpr));
- Map<String, Integer> callCounts = limit.getCallCounts();
+ "expected result to start with "
+ + matchingExpr
+ + " but was "
+ + limit.getCallerMatcher().getTrippedBy(),
+
limit.getCallerMatcher().getTrippedBy().iterator().next().startsWith(matchingExpr));
+ Map<String, Integer> callCounts =
limit.getCallerMatcher().getCallCounts();
assertTrue("call count should be > 0", callCounts.get(matchingExpr) > 0);
}
}
diff --git
a/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java
b/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java
index 0deed2273e9..559a798296b 100644
---
a/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java
+++
b/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java
@@ -17,193 +17,45 @@
package org.apache.solr.search;
import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+import org.apache.solr.util.CallerMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Helper class to simulate query timeouts at specific points in various
components that call {@link
- * QueryLimits#shouldExit()}. These calling points are identified by the
calling class' simple name
- * and optionally a method name and the optional maximum count, e.g.
<code>MoreLikeThisComponent
- * </code> or <code>
- * ClusteringComponent.finishStage</code>,
<code>ClusteringComponent.finishStage:100</code>.
- *
- * <p>NOTE: implementation details cause the expression
<code>simpleName</code> to be disabled when
- * also any <code>simpleName.anyMethod[:NNN]</code> expression is used for the
same class name.
- *
- * <p>NOTE 2: when maximum count is a negative number e.g.
<code>simpleName.someMethod:-1</code>
- * then only the number of calls to {@link QueryLimits#shouldExit()} for that
expression will be
- * reported but no limit will be enforced.
+ * QueryLimits#shouldExit()}. This class uses {@link CallerMatcher} to collect
the matching callers
+ * information and enforce the count limits.
*/
public class CallerSpecificQueryLimit implements QueryLimit {
private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final StackWalker stackWalker =
- StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
- // className -> set of method names
- private final Map<String, Set<String>> interestingCallers = new HashMap<>();
- // expr -> initial count
- private final Map<String, Integer> maxCounts = new ConcurrentHashMap<>();
- // expr -> current count
- private final Map<String, AtomicInteger> callCounts = new
ConcurrentHashMap<>();
- private Set<String> trippedBy = ConcurrentHashMap.newKeySet();
+ private final CallerMatcher callerMatcher;
/**
* Signal a timeout in places that match the calling classes (and methods).
*
* @param callerExprs list of expressions in the format of <code>
- * simpleClassName[.methodName][:NNN]</code>. If the list is empty or
null then the first call
- * to {@link #shouldExit()} from any caller will match.
+ * ( simpleClassName[.methodName] | * )[:NNN]</code>. If the list is
empty or null then the
+ * first call to {@link #shouldExit()} from any caller will match.
*/
- public CallerSpecificQueryLimit(String... callerExprs) {
- this(callerExprs != null ? Arrays.asList(callerExprs) : List.of());
- }
-
public CallerSpecificQueryLimit(Collection<String> callerExprs) {
- for (String callerExpr : callerExprs) {
- String[] exprCount = callerExpr.split(":");
- if (exprCount.length > 2) {
- throw new RuntimeException("Invalid count in callerExpr: " +
callerExpr);
- }
- String[] clazzMethod = exprCount[0].split("\\.");
- if (clazzMethod.length > 2) {
- throw new RuntimeException("Invalid method in callerExpr: " +
callerExpr);
- }
- Set<String> methods =
- interestingCallers.computeIfAbsent(clazzMethod[0], c -> new
HashSet<>());
- if (clazzMethod.length > 1) {
- methods.add(clazzMethod[1]);
- }
- if (exprCount.length > 1) {
- try {
- int count = Integer.parseInt(exprCount[1]);
- maxCounts.put(exprCount[0], count);
- callCounts.put(exprCount[0], new AtomicInteger(0));
- } catch (NumberFormatException e) {
- throw new RuntimeException("Invalid count in callerExpr: " +
callerExpr, e);
- }
- }
- }
- }
-
- /** Returns the set of caller expressions that were tripped. */
- public Set<String> getTrippedBy() {
- return trippedBy;
+ // exclude myself and QueryLimits
+ callerMatcher =
+ new CallerMatcher(
+ callerExprs,
+ Set.of(
+ CallerSpecificQueryLimit.class.getSimpleName(),
QueryLimits.class.getSimpleName()));
}
- /** Returns a map of tripped caller expressions to their current call
counts. */
- public Map<String, Integer> getCallCounts() {
- return callCounts.entrySet().stream()
- .collect(
- Collectors.toMap(
- e ->
- e.getKey()
- + (maxCounts.containsKey(e.getKey())
- ? ":" + maxCounts.get(e.getKey())
- : ""),
- e -> e.getValue().get()));
+ public CallerMatcher getCallerMatcher() {
+ return callerMatcher;
}
@Override
public boolean shouldExit() {
- Optional<String> matchingExpr =
- stackWalker.walk(
- s ->
- s.filter(
- frame -> {
- Class<?> declaring = frame.getDeclaringClass();
- // skip bottom-most frames: myself and QueryLimits
- if (declaring == this.getClass() || declaring ==
QueryLimits.class) {
- return false;
- }
- String method = frame.getMethodName();
- if (interestingCallers.isEmpty()) {
- // any caller is an offending caller
- String expr = declaring.getSimpleName() + "." +
method;
- if (log.isInfoEnabled()) {
- log.info("++++ Limit tripped by any first
caller: {} ++++", expr);
- }
- trippedBy.add(expr);
- callCounts
- .computeIfAbsent(expr, k -> new
AtomicInteger())
- .incrementAndGet();
- return true;
- }
- Set<String> methods =
interestingCallers.get(declaring.getSimpleName());
- if (methods == null) {
- // no class and no methods specified for this
class, so skip
- return false;
- }
- // MATCH. Class name was specified, possibly with
methods.
- // If methods is empty then all methods match,
otherwise only the
- // specified methods match.
- if (methods.isEmpty() || methods.contains(method)) {
- String expr = declaring.getSimpleName();
- if (methods.contains(method)) {
- expr = expr + "." + method;
- } else {
- // even though we don't match/enforce at the
method level, still
- // record the method counts to give better
insight into the callers
- callCounts
- .computeIfAbsent(
- declaring.getSimpleName() + "." + method,
- k -> new AtomicInteger(0))
- .incrementAndGet();
- }
- int currentCount =
- callCounts
- .computeIfAbsent(expr, k -> new
AtomicInteger(0))
- .incrementAndGet();
- // check if we have a max count for this expression
- if (maxCounts.containsKey(expr)) {
- int maxCount = maxCounts.getOrDefault(expr, 0);
- // if max count is negative then just report the
call count
- if (maxCount < 0) {
- maxCount = Integer.MAX_VALUE;
- }
- if (currentCount > maxCount) {
- if (log.isInfoEnabled()) {
- log.info(
- "++++ Limit tripped by caller: {},
current count: {}, max: {} ++++",
- expr,
- currentCount,
- maxCounts.get(expr));
- }
- trippedBy.add(expr + ":" +
maxCounts.get(expr));
- return true;
- } else {
- return false; // max count not reached, not
tripped yet
- }
- } else {
- trippedBy.add(expr);
- if (log.isInfoEnabled()) {
- log.info("++++ Limit tripped by caller: {}
++++", expr);
- }
- return true; // no max count, so tripped on
first call
- }
- } else {
- return false;
- }
- })
- .map(
- frame ->
-
(frame.getDeclaringClass().getSimpleName().isBlank()
- ? frame.getClassName()
- :
frame.getDeclaringClass().getSimpleName())
- + "."
- + frame.getMethodName())
- .findFirst());
- return matchingExpr.isPresent();
+ return callerMatcher.checkCaller().isPresent();
}
@Override
diff --git
a/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.java
b/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.java
new file mode 100644
index 00000000000..22030d357e3
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Helper class to collect interesting callers at specific points. These
calling points are
+ * identified by the calling class' simple name and optionally a method name
and the optional
+ * maximum count, e.g. <code>MoreLikeThisComponent</code> or
<code>ClusteringComponent.finishStage
+ * </code>, <code>ClusteringComponent.finishStage:100</code>. A single
wildcard name <code>*</code>
+ * may be used to mean "any class", which may be useful to e.g. collect all
callers using an
+ * expression <code>*:-1</code>.
+ *
+ * <p>Within your caller you should invoke {@link #checkCaller()} to count any
matching frames in
+ * the current stack, and check if any of the count limits has been reached.
Each invocation will
+ * increase the call count of the matching expression(s). For one invocation
multiple matching
+ * expression counts can be affected because all current stack frames are
examined against the
+ * matching expressions.
+ *
+ * <p>NOTE: implementation details cause the expression
<code>simpleName[:NNN]</code> to be disabled
+ * when also any <code>simpleName.anyMethod[:NNN]</code> expression is used
for the same class name.
+ *
+ * <p>NOTE 2: when maximum count is a negative number e.g.
<code>simpleName[.someMethod]:-1</code>
+ * then only the number of calls to {@link #checkCaller()} for the matching
expressions will be
+ * reported but {@link #checkCaller()} will never return this expression.
+ */
+public class CallerMatcher {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String WILDCARD = "*";
+
+ private final StackWalker stackWalker =
+ StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
+ // className -> set of method names
+ private final Map<String, Set<String>> exactCallers = new HashMap<>();
+ private final Map<String, Set<String>> excludeCallers = new HashMap<>();
+ // expr -> initial count
+ private final Map<String, Integer> maxCounts = new ConcurrentHashMap<>();
+ // expr -> current count
+ private final Map<String, AtomicInteger> callCounts = new
ConcurrentHashMap<>();
+ private Set<String> trippedBy = ConcurrentHashMap.newKeySet();
+
+ /**
+ * Create an instance that reacts to the specified caller expressions.
+ *
+ * @param callerExprs list of expressions in the format of <code>
+ * ( simpleClassName[.methodName] | * )[:NNN]</code>. If the list is
empty or null then the
+ * first call to {@link #checkCaller()} ()} from any caller will match.
+ */
+ public CallerMatcher(Collection<String> callerExprs, Collection<String>
excludeExprs) {
+ for (String callerExpr : callerExprs) {
+ String[] exprCount = callerExpr.split(":");
+ if (exprCount.length > 2) {
+ throw new RuntimeException("Invalid count in callerExpr: " +
callerExpr);
+ }
+ String[] clazzMethod = exprCount[0].split("\\.");
+ if (clazzMethod.length > 2) {
+ throw new RuntimeException("Invalid method in callerExpr: " +
callerExpr);
+ }
+ Set<String> methods = exactCallers.computeIfAbsent(clazzMethod[0], c ->
new HashSet<>());
+ if (clazzMethod.length > 1) {
+ methods.add(clazzMethod[1]);
+ }
+ if (exprCount.length > 1) {
+ try {
+ int count = Integer.parseInt(exprCount[1]);
+ maxCounts.put(exprCount[0], count);
+ callCounts.put(exprCount[0], new AtomicInteger(0));
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Invalid count in callerExpr: " +
callerExpr, e);
+ }
+ }
+ }
+ for (String excludeExpr : excludeExprs) {
+ String[] clazzMethod = excludeExpr.split("\\.");
+ if (clazzMethod.length > 2) {
+ throw new RuntimeException("Invalid method in excludeExpr: " +
excludeExpr);
+ }
+ Set<String> methods = excludeCallers.computeIfAbsent(clazzMethod[0], c
-> new HashSet<>());
+ if (clazzMethod.length > 1) {
+ methods.add(clazzMethod[1]);
+ }
+ }
+ }
+
+ /**
+ * Returns the set of caller expressions that were tripped (reached their
count limit). This
+ * method can be called after {@link #checkCaller()} returns a matching
expression to obtain all
+ * expressions that exceeded their count limits.
+ */
+ public Set<String> getTrippedBy() {
+ return Collections.unmodifiableSet(trippedBy);
+ }
+
+ /** Returns a map of matched caller expressions to their current call
counts. */
+ public Map<String, Integer> getCallCounts() {
+ return callCounts.entrySet().stream()
+ .collect(
+ Collectors.toMap(
+ e ->
+ e.getKey()
+ + (maxCounts.containsKey(e.getKey())
+ ? ":" + maxCounts.get(e.getKey())
+ : ""),
+ e -> e.getValue().get()));
+ }
+
+ /**
+ * Returns the matching caller expression when its count limit was reached,
or empty if no caller
+ * or no count limit was reached. Each invocation increases the call count
of the matching caller,
+ * if any. It's up to the caller to decide whether to continue processing
after this count limit
+ * is reached. The matching expression returned by this call will be also
present in {@link
+ * #getTrippedBy()}.
+ */
+ public Optional<String> checkCaller() {
+ return stackWalker.walk(
+ s ->
+ s.filter(
+ frame -> {
+ // handle exclusions first
+ if (isExcluded(frame)) {
+ return false;
+ }
+
+ String className =
frame.getDeclaringClass().getSimpleName();
+ String method = frame.getMethodName();
+ // now handle the matching expressions
+
+ if (processAnyMatching(className, method)) {
+ return true;
+ }
+
+ Set<String> methods = exactCallers.get(className);
+ boolean wildcardMatch = false;
+ if (methods == null) {
+ // check for wildcard
+ methods = exactCallers.get(WILDCARD);
+ if (methods == null) {
+ // no class and no methods specified for this class,
so skip
+ return false;
+ } else {
+ wildcardMatch = true;
+ }
+ }
+ // MATCH. Class name was specified, possibly with
methods.
+ // If methods is empty then all methods match, otherwise
only the
+ // specified methods match.
+ if (methods.isEmpty() || methods.contains(method)) {
+ String expr = wildcardMatch ? WILDCARD : className;
+ if (methods.contains(method)) {
+ expr = expr + "." + method;
+ } else {
+ // even though we don't match/enforce at the method
level, still
+ // record the method counts to give better insight
into the callers
+ callCounts
+ .computeIfAbsent(className + "." + method, k ->
new AtomicInteger(0))
+ .incrementAndGet();
+ }
+ int currentCount =
+ callCounts
+ .computeIfAbsent(expr, k -> new
AtomicInteger(0))
+ .incrementAndGet();
+ return processMatchWithLimit(expr, currentCount);
+ } else {
+ return false;
+ }
+ })
+ .map(
+ frame ->
+ (frame.getDeclaringClass().getSimpleName().isBlank()
+ ? frame.getClassName()
+ : frame.getDeclaringClass().getSimpleName())
+ + "."
+ + frame.getMethodName())
+ .findFirst());
+ }
+
+ private boolean isExcluded(StackWalker.StackFrame frame) {
+ Class<?> declaring = frame.getDeclaringClass();
+ String method = frame.getMethodName();
+ // always skip myself
+ if (declaring == this.getClass()) {
+ return true;
+ }
+ // skip exclusions, if any
+ Set<String> excludeMethods = excludeCallers.get(declaring.getSimpleName());
+ if (excludeMethods != null) {
+ // skip any method
+ if (excludeMethods.isEmpty()) {
+ return true;
+ } else {
+ // or only the matching method
+ return excludeMethods.contains(method);
+ }
+ }
+ return false;
+ }
+
+ private boolean processAnyMatching(String className, String method) {
+ if (exactCallers.isEmpty()) {
+ // any caller is an offending caller
+ String expr = className + "." + method;
+ if (log.isInfoEnabled()) {
+ log.info("++++ Tripped by any first caller: {} ++++", expr);
+ }
+ trippedBy.add(expr);
+ callCounts.computeIfAbsent(expr, k -> new
AtomicInteger()).incrementAndGet();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean processMatchWithLimit(String expr, int currentCount) {
+ // check if we have a max count for this expression
+ if (maxCounts.containsKey(expr)) {
+ int maxCount = maxCounts.get(expr);
+ // if max count is negative then just report the call count
+ if (maxCount < 0) {
+ maxCount = Integer.MAX_VALUE;
+ }
+ if (currentCount > maxCount) {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "++++ Tripped by caller: {}, current count: {}, max: {} ++++",
+ expr,
+ currentCount,
+ maxCounts.get(expr));
+ }
+ trippedBy.add(expr + ":" + maxCounts.get(expr));
+ return true;
+ } else {
+ return false; // max count not reached, not tripped yet
+ }
+ } else {
+ trippedBy.add(expr);
+ if (log.isInfoEnabled()) {
+ log.info("++++ Tripped by caller: {} ++++", expr);
+ }
+ return true; // no max count, so tripped on first call
+ }
+ }
+}
diff --git
a/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java
b/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java
index da8f58c73cf..5259d3d7480 100644
---
a/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java
+++
b/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java
@@ -115,14 +115,14 @@ public class CallerSpecificQueryLimitTest extends
SolrTestCaseJ4 {
matchingCallCounts.add(matchingClassName + ".doWork");
}
- CallerSpecificQueryLimit limit = new CallerSpecificQueryLimit(callerExpr);
+ CallerSpecificQueryLimit limit = new
CallerSpecificQueryLimit(Set.of(callerExpr));
LimitedWorker limitedWorker = new LimitedWorker(limit);
LimitedWorker2 limitedWorker2 = new LimitedWorker2(limit);
for (int i = 0; i < count * 2; i++) {
limitedWorker2.doWork();
limitedWorker.doWork();
}
- Set<String> trippedBy = limit.getTrippedBy();
+ Set<String> trippedBy = limit.getCallerMatcher().getTrippedBy();
if (shouldTrip) {
assertFalse("Limit should have been tripped, callerExpr: " + callerExpr,
trippedBy.isEmpty());
for (String nonMatchingCallerExpr : nonMatchingCallerExprs) {
@@ -141,7 +141,7 @@ public class CallerSpecificQueryLimitTest extends
SolrTestCaseJ4 {
+ trippedBy,
trippedBy.isEmpty());
}
- Map<String, Integer> callCounts = limit.getCallCounts();
+ Map<String, Integer> callCounts = limit.getCallerMatcher().getCallCounts();
for (String matchingCallCount : matchingCallCounts) {
assertTrue(
"Call count for " + matchingCallCount + " should be > 0, callCounts:
" + callCounts,
