This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 56cc230 SOLR-15564: Improve filtering expressions in /admin/metrics.
56cc230 is described below
commit 56cc23021df974230e8fe62c8af5f95a55fec34f
Author: Andrzej Bialecki <[email protected]>
AuthorDate: Mon Aug 2 17:45:19 2021 +0200
SOLR-15564: Improve filtering expressions in /admin/metrics.
---
solr/CHANGES.txt | 2 +
.../apache/solr/handler/admin/MetricsHandler.java | 101 ++++++++++++++++++++-
.../solr/handler/admin/MetricsHandlerTest.java | 73 +++++++++++++++
solr/solr-ref-guide/src/metrics-reporting.adoc | 33 ++++++-
4 files changed, 203 insertions(+), 6 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4519edb..84502c8 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -385,6 +385,8 @@ Improvements
* SOLR-15570: Include fields declared in the schema in table metadata (SQL)
even if they are empty (Timothy Potter)
+* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)
+
Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno
Roustant)
diff --git
a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 0aa917a..52d90f9 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -21,9 +21,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -66,11 +68,12 @@ public class MetricsHandler extends RequestHandlerBase
implements PermissionName
public static final String REGISTRY_PARAM = "registry";
public static final String GROUP_PARAM = "group";
public static final String KEY_PARAM = "key";
+ public static final String EXPR_PARAM = "expr";
public static final String TYPE_PARAM = "type";
public static final String ALL = "all";
- private static final Pattern KEY_REGEX = Pattern.compile("(?<!" +
Pattern.quote("\\") + ")" + Pattern.quote(":"));
+ private static final Pattern KEY_SPLIT_REGEX = Pattern.compile("(?<!" +
Pattern.quote("\\") + ")" + Pattern.quote(":"));
private final CoreContainer cc;
private final Map<String, String> injectedSysProps =
CommonTestInjection.injectAdditionalProps();
private final boolean enabled;
@@ -109,7 +112,7 @@ public class MetricsHandler extends RequestHandlerBase
implements PermissionName
handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
}
- public void handleRequest(SolrParams params, BiConsumer<String, Object>
consumer) throws Exception {
+ private void handleRequest(SolrParams params, BiConsumer<String, Object>
consumer) throws Exception {
if (!enabled) {
consumer.accept("error", "metrics collection is disabled");
return;
@@ -120,6 +123,11 @@ public class MetricsHandler extends RequestHandlerBase
implements PermissionName
handleKeyRequest(keys, consumer);
return;
}
+ String[] exprs = params.getParams(EXPR_PARAM);
+ if (exprs != null && exprs.length > 0) {
+ handleExprRequest(exprs, consumer);
+ return;
+ }
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
@@ -139,14 +147,95 @@ public class MetricsHandler extends RequestHandlerBase
implements PermissionName
consumer.accept("metrics", response);
}
- public void handleKeyRequest(String[] keys, BiConsumer<String, Object>
consumer) {
+ private static class MetricsExpr {
+ Pattern registryRegex;
+ MetricFilter metricFilter;
+ Predicate<CharSequence> propertyFilter;
+ }
+
+ private void handleExprRequest(String[] exprs, BiConsumer<String, Object>
consumer) {
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+ SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();
+ List<MetricsExpr> metricsExprs = new ArrayList<>();
+
+ for (String key : exprs) {
+ if (key == null || key.isEmpty()) {
+ continue;
+ }
+ String[] parts = KEY_SPLIT_REGEX.split(key);
+ if (parts.length < 2 || parts.length > 3) {
+ errors.add(key, "at least two and at most three colon-separated parts
must be provided");
+ continue;
+ }
+ MetricsExpr me = new MetricsExpr();
+ me.registryRegex = Pattern.compile(unescape(parts[0]));
+ me.metricFilter = new SolrMetricManager.RegexFilter(unescape(parts[1]));
+ String propertyPart = parts.length > 2 ? unescape(parts[2]) : null;
+ if (propertyPart == null) {
+ me.propertyFilter = name -> true;
+ } else {
+ me.propertyFilter = new Predicate<>() {
+ final Pattern pattern = Pattern.compile(propertyPart);
+ @Override
+ public boolean test(CharSequence charSequence) {
+ return pattern.matcher(charSequence).matches();
+ }
+ };
+ }
+ metricsExprs.add(me);
+ }
+ // find matching registries first, to avoid scanning non-matching
registries
+ Set<String> matchingRegistries = new TreeSet<>();
+ metricsExprs.forEach(me -> {
+ metricManager.registryNames().forEach(name -> {
+ if (me.registryRegex.matcher(name).matches()) {
+ matchingRegistries.add(name);
+ }
+ });
+ });
+ for (String registryName : matchingRegistries) {
+ MetricRegistry registry = metricManager.registry(registryName);
+ for (MetricsExpr me : metricsExprs) {
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<Object> perRegistryResult =
(SimpleOrderedMap<Object>) result.get(registryName);
+ final SimpleOrderedMap<Object> perRegistryTemp = new
SimpleOrderedMap<>();
+ // skip processing if not a matching registry
+ if (!me.registryRegex.matcher(registryName).matches()) {
+ continue;
+ }
+ MetricUtils.toMaps(registry,
Collections.singletonList(MetricFilter.ALL), me.metricFilter,
+ me.propertyFilter, false, false, true, false, (k, v) ->
perRegistryTemp.add(k, v));
+ // extracted some metrics and there's no entry for this registry yet
+ if (perRegistryTemp.size() > 0) {
+ if (perRegistryResult == null) { // new results for this registry
+ result.add(registryName, perRegistryTemp);
+ } else {
+ // merge if needed
+ for (Iterator<Map.Entry<String, Object>> it =
perRegistryTemp.iterator(); it.hasNext(); ) {
+ Map.Entry<String, Object> entry = it.next();
+ Object existing = perRegistryResult.get(entry.getKey());
+ if (existing == null) {
+ perRegistryResult.add(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+ consumer.accept("metrics", result);
+ if (errors.size() > 0) {
+ consumer.accept("errors", errors);
+ }
+ }
+
+ private void handleKeyRequest(String[] keys, BiConsumer<String, Object>
consumer) {
SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();
for (String key : keys) {
if (key == null || key.isEmpty()) {
continue;
}
- String[] parts = KEY_REGEX.split(key);
+ String[] parts = KEY_SPLIT_REGEX.split(key);
if (parts.length < 2 || parts.length > 3) {
errors.add(key, "at least two and at most three colon-separated parts
must be provided");
continue;
@@ -200,7 +289,9 @@ public class MetricsHandler extends RequestHandlerBase
implements PermissionName
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '\\') {
- continue;
+ if (i < s.length() - 1 && s.charAt(i + 1) == ':') {
+ continue;
+ }
}
sb.append(c);
}
diff --git
a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 338c38f..adcc305 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import com.codahale.metrics.Counter;
@@ -350,6 +351,78 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
}
@Test
+ @SuppressWarnings("unchecked")
+ public void testExprMetrics() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate";
+ SolrQueryResponse resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics",
CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key1), resp);
+ // response structure is like in the case of non-key params
+ Object val = resp.getValues().findRecursive( "metrics",
"solr.core.collection1", "QUERY./select.requestTimes");
+ assertNotNull(val);
+ assertTrue(val instanceof MapWriter);
+ Map<String, Object> map = new HashMap<>();
+ ((MapWriter) val).toMap(map);
+ assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15
+ assertNotNull(map.toString(), map.get("meanRate"));
+ assertNotNull(map.toString(), map.get("1minRate"));
+ assertNotNull(map.toString(), map.get("5minRate"));
+ assertNotNull(map.toString(), map.get("15minRate"));
+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(),
0.0, 0.0);
+ map.clear();
+
+ String key2 = "solr\\.core\\..*:.*/select\\.request.*";
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics",
CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key2), resp);
+ // response structure is like in the case of non-key params
+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");
+ assertNotNull(val);
+ Object v = ((SimpleOrderedMap<Object>)
val).get("QUERY./select.requestTimes");
+ assertNotNull(v);
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 14, map.size());
+ assertNotNull(map.toString(), map.get("1minRate"));
+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(),
0.0, 0.0);
+ map.clear();
+ // select requests counter
+ v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests");
+ assertNotNull(v);
+ assertTrue(v instanceof Number);
+
+ // test multiple expressions producing overlapping metrics - should be no
dupes
+
+ // this key matches also sub-metrics of /select, eg. /select.distrib,
/select.local, ...
+ String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";
+ resp = new SolrQueryResponse();
+ // ORDER OF PARAMS MATTERS HERE! see the refguide
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics",
CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key2, MetricsHandler.EXPR_PARAM, key1,
MetricsHandler.EXPR_PARAM, key3), resp);
+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");
+ assertNotNull(val);
+ // for requestTimes only the full set of values from the first expr should
be present
+ assertNotNull(val);
+ SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;
+ assertEquals(values.jsonStr(), 4, values.size());
+ List<Object> multipleVals = values.getAll("QUERY./select.requestTimes");
+ assertEquals(multipleVals.toString(), 1, multipleVals.size());
+ v = values.get("QUERY./select.local.requestTimes");
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 1, map.size());
+ assertTrue(map.toString(), map.containsKey("count"));
+ map.clear();
+ v = values.get("QUERY./select.distrib.requestTimes");
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 1, map.size());
+ assertTrue(map.toString(), map.containsKey("count"));
+ }
+
+ @Test
public void testMetricsUnload() throws Exception {
SolrCore core =
h.getCoreContainer().getCore("collection1");//;.getRequestHandlers().put("/dumphandler",
new DumpRequestHandler());
diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc
b/solr/solr-ref-guide/src/metrics-reporting.adoc
index 8854be7..59326fa 100644
--- a/solr/solr-ref-guide/src/metrics-reporting.adoc
+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc
@@ -763,7 +763,33 @@ Examples:
* `key=solr.jvm:system.properties:user.name`
+
-*NOTE: when this parameter is used, other selection methods listed above are
ignored.*
+*NOTE: when this parameter is used, any other selection methods are ignored.*
+
+`expr`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: none
+|===
++
+Extended notation of the `key` selection criteria, which supports regular
expressions for each of the
+parts supported by the `key` selector. This parameter can be specified
multiple times to retrieve metrics that match
+any expression. The API guarantees that the output will consist only of unique
metric names even if
+multiple expressions match the same metric name. Note: order of multiple
`expr` parameters matters here
+- only the first value of the first matching expression will be recorded,
subsequent values for the same metric name
+produced by matching other expressions will be skipped.
++
+Fully-qualified expression consists of at least two and at most three regex
patterns separated by
+colons: a registry pattern, colon, a metric pattern, and then an optional
colon and metric property pattern.
+Colons and other regex meta-characters in names and in regular expressions
MUST be escaped using backslash (`\`) character.
+
+Examples:
+
+* `expr=solr\.core\..*:QUERY\..*\.requestTimes:max_ms`
+* `expr=solr\.jvm:system\.properties:user\..*`
+
++
+*NOTE: when this parameter is used, any other selection methods are ignored.*
`compact`::
+
@@ -845,3 +871,8 @@ Request only "user.name" property of "system.properties"
metric from registry "s
[source,text]
http://localhost:8983/solr/admin/metrics?wt=xml&key=solr.jvm:system.properties:user.name
+
+Request query rates (but not histograms) from any core in any collection in
any QUERY handler:
+
+[source,text]
+http://localhost:8983/solr/admin/metrics?expr=solr\.core\..*:QUERY\..*\.requestTimes:.*Rate
