This is an automated email from the ASF dual-hosted git repository. mlbiscoc pushed a commit to branch feature/SOLR-17458-rebased in repository https://gitbox.apache.org/repos/asf/solr.git
commit 046b206de34417bb672b335fe607d90dda586e39 Author: Matthew Biscocho <[email protected]> AuthorDate: Fri Sep 5 12:57:31 2025 -0400 SOLR-17854: Support filters to /admin/metrics endpoint for Prometheus metrics (#3499) * Add fitlering for /admin/metrics * Rename vars * Use a generic method for handling the 3 types metric types * Optimize requiredLabelsFilter * Add tests for MetricsHandlerTest * Add tests around FilterablePrometheusMetricReader * static member * Fix tests --------- Co-authored-by: David Smiley <[email protected]> Co-authored-by: Matthew Biscocho <[email protected]> --- .../src/java/org/apache/solr/core/SolrCore.java | 15 +- .../apache/solr/handler/admin/MetricsHandler.java | 159 ++++- .../org/apache/solr/metrics/SolrMetricManager.java | 12 +- .../otel/FilterablePrometheusMetricReader.java | 185 +++++ .../solr/response/PrometheusResponseWriter.java | 113 +--- .../solr/handler/admin/MetricsHandlerTest.java | 751 ++++----------------- .../otel/FilterablePrometheusMetricReaderTest.java | 95 +++ .../apache/solr/update/SolrIndexMetricsTest.java | 2 +- 8 files changed, 586 insertions(+), 746 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index ab4db7d04f2..a8cb4800fde 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1422,14 +1422,13 @@ public class SolrCore implements SolrInfoBean, Closeable { }), OtelUnit.BYTES)); - observables.add( - parentContext.observableLongGauge( - "solr_core_segment_count", - "Number of segments in a Solr core", - (observableLongMeasurement -> { - if (isReady()) - observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes); - }))); + parentContext.observableLongGauge( + "solr_core_segments", + "Number of segments in a Solr core", + (observableLongMeasurement -> { + if (isReady()) + observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes); + })); // NOCOMMIT: Do we need these start_time metrics? I think at minimum it should be optional // otherwise we fall into metric bloat for something people may not care about. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java index 633ccc7bb00..143cf453f41 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java @@ -25,14 +25,23 @@ import com.codahale.metrics.Metric; import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.InfoSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiConsumer; import java.util.function.Predicate; @@ -49,6 +58,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; @@ -69,6 +79,16 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName public static final String KEY_PARAM = "key"; public static final String EXPR_PARAM = "expr"; public static final String TYPE_PARAM = "type"; + // Prometheus filtering parameters + public static final String CATEGORY_PARAM = "category"; + public static final String CORE_PARAM = "core"; + public static final String COLLECTION_PARAM = "collection"; + public static final String SHARD_PARAM = "shard"; + public static final String REPLICA_PARAM = "replica"; + public static final String METRIC_NAME_PARAM = "name"; + private static final Set<String> labelFilterKeys = + Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_PARAM); + // NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metrics public static final String PROMETHEUS_METRICS_WT = "prometheus"; public static final String OPEN_METRICS_WT = "openmetrics"; @@ -131,7 +151,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName // NOCOMMIT SOLR-17458: Make this the default option after dropwizard removal if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT)) || OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) { - consumer.accept("metrics", metricManager.getPrometheusMetricReaders()); + handlePrometheusRequest(params, consumer); return; } @@ -151,6 +171,58 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName consumer.accept("metrics", response); } + private void handlePrometheusRequest(SolrParams params, BiConsumer<String, Object> consumer) { + Set<String> metricNames = readParamsAsSet(params, METRIC_NAME_PARAM); + SortedMap<String, Set<String>> labelFilters = labelFilters(params); + + if (metricNames.isEmpty() && labelFilters.isEmpty()) { + consumer.accept( + "metrics", + mergeSnapshots( + metricManager.getPrometheusMetricReaders().values().stream() + .flatMap(r -> r.collect().stream()) + .toList())); + return; + } + + List<MetricSnapshot> allSnapshots = new ArrayList<>(); + for (FilterablePrometheusMetricReader reader : + metricManager.getPrometheusMetricReaders().values()) { + MetricSnapshots filteredSnapshots = reader.collect(metricNames, labelFilters); + filteredSnapshots.forEach(allSnapshots::add); + } + + // Merge all filtered snapshots and return the merged result + MetricSnapshots mergedSnapshots = mergeSnapshots(allSnapshots); + consumer.accept("metrics", mergedSnapshots); + } + + private SortedMap<String, Set<String>> labelFilters(SolrParams params) { + SortedMap<String, Set<String>> labelFilters = new TreeMap<>(); + labelFilterKeys.forEach( + (paramName) -> { + Set<String> filterValues = readParamsAsSet(params, paramName); + if (!filterValues.isEmpty()) { + labelFilters.put(paramName, filterValues); + } + }); + + return labelFilters; + } + + private Set<String> readParamsAsSet(SolrParams params, String paramName) { + String[] paramValues = params.getParams(paramName); + if (paramValues == null || paramValues.length == 0) { + return Set.of(); + } + + List<String> paramSet = new ArrayList<>(); + for (String param : paramValues) { + paramSet.addAll(StrUtils.splitSmart(param, ',')); + } + return Set.copyOf(paramSet); + } + private NamedList<Object> handleDropwizardRegistry(SolrParams params) { boolean compact = params.getBool(COMPACT_PARAM, true); MetricFilter mustMatchFilter = parseMustMatchFilter(params); @@ -182,6 +254,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName return response; } + // NOCOMMIT: Remove this filtering logic private static class MetricsExpr { Pattern registryRegex; MetricFilter metricFilter; @@ -499,9 +572,87 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName return metricTypes; } - private String getCoreNameFromRegistry(String registryName) { - String coreName = registryName.substring(registryName.indexOf('.') + 1); - return coreName.replace(".", "_"); + /** + * Merge a collection of individual {@link MetricSnapshot} instances into one {@link + * MetricSnapshots}. This is necessary because we create a {@link + * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric + * names across cores which is an illegal format if under the same prometheus grouping. + */ + private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) { + Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>(); + Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>(); + Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>(); + InfoSnapshot otelInfoSnapshots = null; + + for (MetricSnapshot snapshot : snapshots) { + String metricName = snapshot.getMetadata().getPrometheusName(); + + switch (snapshot) { + case CounterSnapshot counterSnapshot -> { + CounterSnapshot.Builder builder = + counterSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + CounterSnapshot.builder() + .name(counterSnapshot.getMetadata().getName()) + .help(counterSnapshot.getMetadata().getHelp()); + return counterSnapshot.getMetadata().hasUnit() + ? base.unit(counterSnapshot.getMetadata().getUnit()) + : base; + }); + counterSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case GaugeSnapshot gaugeSnapshot -> { + GaugeSnapshot.Builder builder = + gaugeSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + GaugeSnapshot.builder() + .name(gaugeSnapshot.getMetadata().getName()) + .help(gaugeSnapshot.getMetadata().getHelp()); + return gaugeSnapshot.getMetadata().hasUnit() + ? base.unit(gaugeSnapshot.getMetadata().getUnit()) + : base; + }); + gaugeSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case HistogramSnapshot histogramSnapshot -> { + HistogramSnapshot.Builder builder = + histogramSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + HistogramSnapshot.builder() + .name(histogramSnapshot.getMetadata().getName()) + .help(histogramSnapshot.getMetadata().getHelp()); + return histogramSnapshot.getMetadata().hasUnit() + ? base.unit(histogramSnapshot.getMetadata().getUnit()) + : base; + }); + histogramSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case InfoSnapshot infoSnapshot -> { + // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate + // metric called target_info containing OTEL SDK metadata. Only one of these need to be + // kept + if (otelInfoSnapshots == null) + otelInfoSnapshots = + new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints()); + } + default -> { + // Handle unexpected snapshot types gracefully + } + } + } + + MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder(); + counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots); + return snapshotsBuilder.build(); } @Override diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index bda5fa4af39..4ddc6443f80 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -52,7 +52,6 @@ import io.opentelemetry.api.metrics.ObservableLongGauge; import io.opentelemetry.api.metrics.ObservableLongMeasurement; import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; import io.opentelemetry.api.metrics.ObservableMeasurement; -import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter; @@ -86,6 +85,7 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.logging.MDCLoggingContext; +import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader; import org.apache.solr.metrics.otel.OtelUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -753,7 +753,7 @@ public class SolrMetricManager { .computeIfAbsent( providerName, key -> { - var reader = new PrometheusMetricReader(true, null); + var reader = new FilterablePrometheusMetricReader(true, null); // NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP // with an exporter var provider = SdkMeterProvider.builder().registerMetricReader(reader); @@ -1640,17 +1640,17 @@ public class SolrMetricManager { return metricsConfig; } - /** Get a shallow copied map of {@link PrometheusMetricReader}. */ - public Map<String, PrometheusMetricReader> getPrometheusMetricReaders() { + /** Get a shallow copied map of {@link FilterablePrometheusMetricReader}. */ + public Map<String, FilterablePrometheusMetricReader> getPrometheusMetricReaders() { return meterProviderAndReaders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().prometheusMetricReader())); } - public PrometheusMetricReader getPrometheusMetricReader(String providerName) { + public FilterablePrometheusMetricReader getPrometheusMetricReader(String providerName) { MeterProviderAndReaders mpr = meterProviderAndReaders.get(enforcePrefix(providerName)); return (mpr != null) ? mpr.prometheusMetricReader() : null; } private record MeterProviderAndReaders( - SdkMeterProvider sdkMeterProvider, PrometheusMetricReader prometheusMetricReader) {} + SdkMeterProvider sdkMeterProvider, FilterablePrometheusMetricReader prometheusMetricReader) {} } diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java new file mode 100644 index 00000000000..4cd321c112f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel; + +import static java.util.stream.Collectors.toList; + +import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.InfoSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FilterablePrometheusMetricReader extends PrometheusMetricReader { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Set<String> PROM_SUFFIXES = + Set.of("_total", "_sum", "_count", "_bucket", "_gcount", "_gsum", "_created", "_info"); + + public FilterablePrometheusMetricReader( + boolean otelScopeEnabled, Predicate<String> allowedResourceAttributesFilter) { + super(otelScopeEnabled, allowedResourceAttributesFilter); + } + + /** + * Collect metrics with filtering support for metric names and labels. + * + * @param includedNames Set of metric names to include. If empty, all metric names are included. + * @param requiredLabels Map of label names to their allowed values. If empty, no label filtering + * is applied. + * @return Filtered MetricSnapshots + */ + public MetricSnapshots collect( + Set<String> includedNames, SortedMap<String, Set<String>> requiredLabels) { + + // If no filtering is requested then return all metrics + if (includedNames.isEmpty() && requiredLabels.isEmpty()) { + return super.collect(); + } + + // Prometheus appends a suffix to the metrics depending on the metric type. We need to sanitize + // the suffix off if they filter by Prometheus name instead of OTEL name. + Set<String> sanitizedNames = + includedNames.stream() + .map( + (name) -> { + for (String suffix : PROM_SUFFIXES) { + if (name.endsWith(suffix)) { + return name.substring(0, name.lastIndexOf(suffix)); + } + } + return name; + }) + .collect(Collectors.toSet()); + + MetricSnapshots snapshotsToFilter; + if (sanitizedNames.isEmpty()) { + snapshotsToFilter = super.collect(); + } else { + snapshotsToFilter = super.collect(sanitizedNames::contains); + } + + // Return named filtered snapshots if not label filters provided + if (requiredLabels.isEmpty()) { + return snapshotsToFilter; + } + + MetricSnapshots.Builder filteredSnapshots = MetricSnapshots.builder(); + for (MetricSnapshot metricSnapshot : snapshotsToFilter) { + switch (metricSnapshot) { + case CounterSnapshot snapshot -> { + List<CounterSnapshot.CounterDataPointSnapshot> filtered = + filterDatapoint( + snapshot, requiredLabels, CounterSnapshot.CounterDataPointSnapshot.class); + if (!filtered.isEmpty()) { + filteredSnapshots.metricSnapshot(new CounterSnapshot(snapshot.getMetadata(), filtered)); + } + } + case HistogramSnapshot snapshot -> { + List<HistogramSnapshot.HistogramDataPointSnapshot> filtered = + filterDatapoint( + snapshot, requiredLabels, HistogramSnapshot.HistogramDataPointSnapshot.class); + if (!filtered.isEmpty()) { + filteredSnapshots.metricSnapshot( + new HistogramSnapshot(snapshot.getMetadata(), filtered)); + } + } + case GaugeSnapshot snapshot -> { + List<GaugeSnapshot.GaugeDataPointSnapshot> filtered = + filterDatapoint(snapshot, requiredLabels, GaugeSnapshot.GaugeDataPointSnapshot.class); + if (!filtered.isEmpty()) { + filteredSnapshots.metricSnapshot(new GaugeSnapshot(snapshot.getMetadata(), filtered)); + } + } + case InfoSnapshot ignored -> { + // Do nothing for InfoSnapshots. Always filter it out + } + default -> { + log.error("Unknown metric snapshot type {}", metricSnapshot.getClass()); + } + } + } + return filteredSnapshots.build(); + } + + private <D extends DataPointSnapshot> List<D> filterDatapoint( + MetricSnapshot snapshot, + SortedMap<String, Set<String>> requiredLabels, + Class<D> dataPointClass) { + return snapshot.getDataPoints().stream() + .filter(dp -> requiredLabelsFilter(dp.getLabels(), requiredLabels)) + .map(dataPointClass::cast) + .collect(toList()); + } + + static boolean requiredLabelsFilter( + Labels labels, SortedMap<String, Set<String>> requiredLabels) { + // Both Labels and requiredLabels are name-sorted with unique names. + // For each required label, scan forward through labels until we find it. + int labelIdx = 0; + requireLoop: + for (Map.Entry<String, Set<String>> entry : requiredLabels.entrySet()) { + String requiredLabelName = entry.getKey(); + Set<String> allowedValues = entry.getValue(); + + // Labels are sorted, so we can continue from the last position + while (labelIdx < labels.size()) { + String labelName = labels.getName(labelIdx); + + int comparison = labelName.compareTo(requiredLabelName); + + if (comparison < 0) { + // a label before requiredLabelName; we don't care about this one + assert !requiredLabels.containsKey(labelName); + labelIdx++; + continue; // inspect the next input label + } + if (comparison > 0) { + // We've passed where requiredLabelName should be, so it doesn't exist + assert !labels.contains(requiredLabelName); + return false; + } + + assert labels.contains(requiredLabelName); + if (!allowedValues.contains(labels.getValue(labelIdx))) { + return false; + } + // we satisfied requiredLabelName with its allowedValues. Move onto next + labelIdx++; + continue requireLoop; + } // end while loop on input Labels + // didn't find requiredLabelName; ran out of names + assert !labels.contains(requiredLabelName); + return false; + } + return true; // found all requiredLabels in Labels + } +} diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java index 070ed662dc0..a5f69e1632b 100644 --- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java @@ -18,27 +18,14 @@ package org.apache.solr.response; import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT; -import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; -import io.prometheus.metrics.model.snapshots.CounterSnapshot; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.HistogramSnapshot; -import io.prometheus.metrics.model.snapshots.InfoSnapshot; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.io.IOException; import java.io.OutputStream; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.apache.solr.common.params.CommonParams; import org.apache.solr.handler.admin.MetricsHandler; import org.apache.solr.request.SolrQueryRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */ @SuppressWarnings(value = "unchecked") @@ -49,23 +36,17 @@ public class PrometheusResponseWriter implements QueryResponseWriter { private static final String CONTENT_TYPE_OPEN_METRICS = "application/openmetrics-text; version=1.0.0; charset=utf-8"; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Override public void write( OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType) throws IOException { - Map<String, PrometheusMetricReader> readers = - (Map<String, PrometheusMetricReader>) response.getValues().get("metrics"); - - List<MetricSnapshot> snapshots = - readers.values().stream().flatMap(r -> r.collect().stream()).toList(); - + var metrics = response.getValues().get("metrics"); + MetricSnapshots snapshots = (MetricSnapshots) metrics; if (writeOpenMetricsFormat(request)) { - new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots)); + new OpenMetricsTextFormatWriter(false, true).write(out, snapshots); } else { - new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots)); + new PrometheusTextFormatWriter(false).write(out, snapshots); } } @@ -92,90 +73,4 @@ public class PrometheusResponseWriter implements QueryResponseWriter { return acceptHeader.contains("application/openmetrics-text") && (acceptHeader.contains("version=1.0.0")); } - - /** - * Merge a collection of individual {@link MetricSnapshot} instances into one {@link - * MetricSnapshots}. This is necessary because we create a {@link SdkMeterProvider} per Solr core - * resulting in duplicate metric names across cores which is an illegal format if not under the - * same prometheus grouping. - */ - private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) { - Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>(); - Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>(); - Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>(); - InfoSnapshot otelInfoSnapshots = null; - - for (MetricSnapshot snapshot : snapshots) { - String metricName = snapshot.getMetadata().getPrometheusName(); - - switch (snapshot) { - case CounterSnapshot counterSnapshot -> { - CounterSnapshot.Builder builder = - counterSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - CounterSnapshot.builder() - .name(counterSnapshot.getMetadata().getName()) - .help(counterSnapshot.getMetadata().getHelp()); - return counterSnapshot.getMetadata().hasUnit() - ? base.unit(counterSnapshot.getMetadata().getUnit()) - : base; - }); - counterSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case GaugeSnapshot gaugeSnapshot -> { - GaugeSnapshot.Builder builder = - gaugeSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - GaugeSnapshot.builder() - .name(gaugeSnapshot.getMetadata().getName()) - .help(gaugeSnapshot.getMetadata().getHelp()); - return gaugeSnapshot.getMetadata().hasUnit() - ? base.unit(gaugeSnapshot.getMetadata().getUnit()) - : base; - }); - gaugeSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case HistogramSnapshot histogramSnapshot -> { - HistogramSnapshot.Builder builder = - histogramSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - HistogramSnapshot.builder() - .name(histogramSnapshot.getMetadata().getName()) - .help(histogramSnapshot.getMetadata().getHelp()); - return histogramSnapshot.getMetadata().hasUnit() - ? base.unit(histogramSnapshot.getMetadata().getUnit()) - : base; - }); - histogramSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case InfoSnapshot infoSnapshot -> { - // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate - // metric called target_info containing OTEL SDK metadata. Only one of these need to be - // kept - if (otelInfoSnapshots == null) - otelInfoSnapshots = - new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints()); - } - default -> { - log.warn( - "Unexpected snapshot type: {} for metric {}", - snapshot.getClass().getName(), - snapshot.getMetadata().getName()); - } - } - } - - MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder(); - counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots); - return snapshotsBuilder.build(); - } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java index 0878bf2c8cd..6cb09d3d95d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java @@ -17,33 +17,19 @@ package org.apache.solr.handler.admin; -import com.codahale.metrics.Counter; import io.opentelemetry.api.common.Attributes; -import io.prometheus.metrics.model.snapshots.CounterSnapshot; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.Labels; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; -import io.prometheus.metrics.model.snapshots.SummarySnapshot; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.MapWriter; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.core.PluginBag; -import org.apache.solr.core.PluginInfo; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -51,699 +37,228 @@ import org.junit.Test; public class MetricsHandlerTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig-minimal.xml", "schema.xml"); h.getCoreContainer().waitForLoadingCoresToFinish(30000); - - // manually register & seed some metrics in solr.jvm and solr.jetty for testing via handler - // (use "solrtest_" prefix just in case the jvm or jetty adds a "foo" metric at some point) - Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "solrtest_foo"); - c.inc(); - c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo"); - c.inc(2); - // test escapes - c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo:bar"); - c.inc(3); - - h.getCoreContainer() - .getMetricManager() - .meter(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses"); - h.getCoreContainer() - .getMetricManager() - .counter( - null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.active-requests"); - h.getCoreContainer() - .getMetricManager() - .timer(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.dispatches"); - } - - @AfterClass - public static void cleanupMetrics() { - if (null != h) { - h.getCoreContainer().getMetricManager().registry("solr.jvm").remove("solrtest_foo"); - h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo"); - h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar"); - } } - // NOCOMMIT: This test does a bunch of /admin/metrics calls with various params, with various - // filters and parameters. We have not migrated all the metrics to otel yet or even created any - // filters. Once that is done, we should revisit this test and assert the prometheus response. @Test - @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458";) - public void test() throws Exception { + public void testMetricNamesFiltering() throws Exception { + String expectedRequestsMetricName = "solr_core_requests"; MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); + assertQ(req("*:*"), "//result[@numFound='0']"); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", CommonParams.WT, - "json"), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.METRIC_NAME_PARAM, + expectedRequestsMetricName), resp); - NamedList<?> values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1"); - assertNotNull(nl); - Object o = nl.get("SEARCHER.new.errors"); - assertNotNull(o); // counter type - assertTrue(o instanceof MapWriter); - // response wasn't serialized, so we get here whatever MetricUtils produced instead of NamedList - assertNotNull(((MapWriter) o)._get("count")); - assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count")); - assertNotNull(nl.get("INDEX.segments")); // int gauge - assertTrue((int) ((MapWriter) nl.get("INDEX.segments"))._get("value") >= 0); - assertNotNull(nl.get("INDEX.sizeInBytes")); // long gauge - assertTrue((long) ((MapWriter) nl.get("INDEX.sizeInBytes"))._get("value") >= 0); - nl = (NamedList<?>) values.get("solr.node"); - assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge - assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value")); - assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type - Map<String, Object> map = new HashMap<>(); - ((MapWriter) nl.get("ADMIN./admin/authorization.clientErrors")).toMap(map); - assertEquals(5, map.size()); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "jvm,jetty"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(2, values.size()); - assertNotNull(values.get("solr.jetty")); - assertNotNull(values.get("solr.jvm")); - - resp = new SolrQueryResponse(); - // "collection" works too, because it's a prefix for "collection1" - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "registry", - "solr.core.collection,solr.jvm"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(2, values.size()); - assertNotNull(values.get("solr.core.collection1")); - assertNotNull(values.get("solr.jvm")); - - resp = new SolrQueryResponse(); - // "collection" works too, because it's a prefix for "collection1" - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "registry", - "solr.core.collection", - "registry", - "solr.jvm"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(2, values.size()); - assertNotNull(values.get("solr.core.collection1")); - assertNotNull(values.get("solr.jvm")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "jvm,jetty"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(2, values.size()); - assertNotNull(values.get("solr.jetty")); - assertNotNull(values.get("solr.jvm")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "jvm", - "group", - "jetty"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(2, values.size()); - assertNotNull(values.get("solr.jetty")); - assertNotNull(values.get("solr.jvm")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "node", - "type", - "counter"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(1, values.size()); - values = (NamedList<?>) values.get("solr.node"); - assertNotNull(values); - assertNull(values.get("ADMIN./admin/authorization.errors")); // this is a timer node - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "prefix", - "CONTAINER.cores,CONTAINER.threadPool"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(1, values.size()); - assertNotNull(values.get("solr.node")); - values = (NamedList<?>) values.get("solr.node"); - assertEquals(15, values.size()); - assertNotNull(values.get("CONTAINER.cores.lazy")); // this is a gauge node - assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "prefix", - "CONTAINER.cores", - "regex", - "C.*thread.*completed"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertNotNull(values.get("solr.node")); - values = (NamedList<?>) values.get("solr.node"); - assertEquals(5, values.size()); - assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - "prefix", - "CACHE.core.fieldCache", - "property", - "entries_count", - MetricsHandler.COMPACT_PARAM, - "true"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertNotNull(values.get("solr.core.collection1")); - values = (NamedList<?>) values.get("solr.core.collection1"); - assertEquals(1, values.size()); - MapWriter writer = (MapWriter) values.get("CACHE.core.fieldCache"); - assertNotNull(writer); - assertNotNull(writer._get("entries_count")); - - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "jvm", - "prefix", - "CONTAINER.cores"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - assertEquals(0, values.size()); + var metrics = resp.getValues().get("metrics"); + MetricSnapshots snapshots = (MetricSnapshots) metrics; + assertEquals(1, snapshots.size()); + assertEquals(expectedRequestsMetricName, snapshots.get(0).getMetadata().getPrometheusName()); - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - MetricsHandler.COMPACT_PARAM, - "false", - CommonParams.WT, - "json", - "group", - "node", - "type", - "timer", - "prefix", - "CONTAINER.cores"), - resp); - values = resp.getValues(); - assertNotNull(values.get("metrics")); - SimpleOrderedMap<?> map1 = (SimpleOrderedMap<?>) values.get("metrics"); - assertEquals(0, map1.size()); handler.close(); } @Test - public void testPropertyFilter() throws Exception { - assertQ(req("*:*"), "//result[@numFound='0']"); - + public void testMultipleMetricNamesFiltering() throws Exception { + String expectedRequestsMetricName = "solr_core_requests"; + String expectedSearcherMetricName = "solr_core_searcher_new"; + var expected = Set.of(expectedRequestsMetricName, expectedSearcherMetricName); MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); + assertQ(req("*:*"), "//result[@numFound='0']"); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.COMPACT_PARAM, - "true", - "group", - "core", - "prefix", - "CACHE.searcher"), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.METRIC_NAME_PARAM, + expectedRequestsMetricName + "," + expectedSearcherMetricName), resp); - NamedList<?> values = resp.getValues(); - assertNotNull(values.get("metrics")); - values = (NamedList<?>) values.get("metrics"); - NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1"); - assertNotNull(nl); - assertTrue(nl.size() > 0); - nl.forEach( - (k, v) -> { - assertTrue(v instanceof MapWriter); - Map<String, Object> map = new HashMap<>(); - ((MapWriter) v).toMap(map); - assertTrue(map.size() > 2); - }); - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.COMPACT_PARAM, - "true", - "group", - "core", - "prefix", - "CACHE.searcher", - "property", - "inserts", - "property", - "size"), - resp); - values = resp.getValues(); - values = (NamedList<?>) values.get("metrics"); - nl = (NamedList<?>) values.get("solr.core.collection1"); - assertNotNull(nl); - assertTrue(nl.size() > 0); - nl.forEach( - (k, v) -> { - assertTrue(v instanceof MapWriter); - Map<String, Object> map = new HashMap<>(); - ((MapWriter) v).toMap(map); - assertEquals("k=" + k + ", v=" + map, 2, map.size()); - assertNotNull(map.get("inserts")); - assertNotNull(map.get("size")); - }); + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + assertEquals(2, metrics.size()); + Set<String> actual = + metrics.stream().map(m -> m.getMetadata().getPrometheusName()).collect(Collectors.toSet()); + assertEquals(expected, actual); + handler.close(); } - // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet @Test - @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458";) - public void testKeyMetrics() throws Exception { + public void testNonExistentMetricNameFiltering() throws Exception { + String nonexistentMetricName = "nonexistent_metric_name"; MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); - String key1 = "solr.core.collection1:CACHE.core.fieldCache"; SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - key1), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.METRIC_NAME_PARAM, + nonexistentMetricName), resp); - NamedList<?> values = resp.getValues(); - Object val = values._get(List.of("metrics", key1), null); - assertNotNull(val); - assertTrue(val instanceof MapWriter); - assertTrue(((MapWriter) val)._size() >= 2); - - String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count"; - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - key2), - resp); - val = resp.getValues()._get("metrics/" + key2); - assertNotNull(val); - assertTrue(val instanceof Number); + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + assertEquals(0, metrics.size()); + handler.close(); + } - String key3 = "solr.jetty:solrtest_foo\\:bar"; - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - key3), - resp); + @Test + public void testLabelFiltering() throws Exception { + MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); - val = resp.getValues()._get("metrics/" + key3); - assertNotNull(val); - assertTrue(val instanceof Number); - assertEquals(3, ((Number) val).intValue()); + assertQ(req("*:*"), "//result[@numFound='0']"); - // test multiple keys - resp = new SolrQueryResponse(); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - key1, - MetricsHandler.KEY_PARAM, - key2, - MetricsHandler.KEY_PARAM, - key3), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.CATEGORY_PARAM, + "QUERY"), resp); + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + + metrics.forEach( + (ms) -> { + ms.getDataPoints() + .forEach( + (dp) -> { + assertEquals("QUERY", dp.getLabels().get(MetricsHandler.CATEGORY_PARAM)); + }); + }); - val = resp.getValues()._get("metrics/" + key1); - assertNotNull(val); - val = resp.getValues()._get("metrics/" + key2); - assertNotNull(val); - val = resp.getValues()._get("metrics/" + key3); - assertNotNull(val); + handler.close(); + } - String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate"; - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - key4), - resp); - // the key contains a slash, need explicit list of path elements - val = resp.getValues()._get(Arrays.asList("metrics", key4), null); - assertNotNull(val); - assertTrue(val instanceof Number); + @Test + public void testMultipleLabelFiltering() throws Exception { + MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); - // test errors + assertQ(req("*:*"), "//result[@numFound='0']"); - // invalid keys - resp = new SolrQueryResponse(); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - "foo", - MetricsHandler.KEY_PARAM, - "foo:bar:baz:xyz"), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.CATEGORY_PARAM, + "QUERY" + "," + "SEARCHER"), resp); - values = resp.getValues(); - NamedList<?> metrics = (NamedList<?>) values.get("metrics"); - assertEquals(0, metrics.size()); - assertNotNull(values._get(List.of("errors", "foo"), null)); - assertNotNull(values._get(List.of("errors", "foo:bar:baz:xyz"), null)); - // unknown registry - resp = new SolrQueryResponse(); + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + metrics.forEach( + (ms) -> { + ms.getDataPoints() + .forEach( + (dp) -> { + assertTrue( + dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("QUERY") + || dp.getLabels() + .get(MetricsHandler.CATEGORY_PARAM) + .equals("SEARCHER")); + }); + }); + + handler.close(); + } + + @Test + public void testNonExistentLabelFiltering() throws Exception { + MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); + + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - "foo:bar:baz"), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.CORE_PARAM, + "nonexistent_core_name"), resp); - values = resp.getValues(); - metrics = (NamedList<?>) values.get("metrics"); + + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); assertEquals(0, metrics.size()); - assertNotNull(values._get(List.of("errors", "foo:bar:baz"), null)); + handler.close(); + } + + @Test + public void testMixedLabelFiltering() throws Exception { + MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); + + assertQ(req("*:*"), "//result[@numFound='0']"); - // unknown metric - resp = new SolrQueryResponse(); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.KEY_PARAM, - "solr.jetty:unknown:baz"), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.CORE_PARAM, + "collection1", + MetricsHandler.CATEGORY_PARAM, + "SEARCHER"), resp); - values = resp.getValues(); - metrics = (NamedList<?>) values.get("metrics"); - assertEquals(0, metrics.size()); - assertNotNull(values._get(List.of("errors", "solr.jetty:unknown:baz"), null)); + + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + metrics.forEach( + (ms) -> { + ms.getDataPoints() + .forEach( + (dp) -> { + assertTrue( + dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("SEARCHER") + && dp.getLabels().get(MetricsHandler.CORE_PARAM).equals("collection1")); + }); + }); handler.close(); } - // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet @Test - @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458";) - @SuppressWarnings("unchecked") - public void testExprMetrics() throws Exception { + public void testMetricNamesAndLabelFiltering() throws Exception { + String expectedMetricName = "solr_core_segments"; MetricsHandler handler = new MetricsHandler(h.getCoreContainer()); - String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate"; + assertQ(req("*:*"), "//result[@numFound='0']"); + SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequestBody( req( CommonParams.QT, "/admin/metrics", CommonParams.WT, - "json", - MetricsHandler.EXPR_PARAM, - key1), - resp); - // response structure is like in the case of non-key params - Object val = - resp.getValues() - ._get(List.of("metrics", "solr.core.collection1", "QUERY./select.requestTimes"), null); - assertNotNull(val); - assertTrue(val instanceof MapWriter); - Map<String, Object> map = new HashMap<>(); - ((MapWriter) val).toMap(map); - assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15 - assertNotNull(map.toString(), map.get("meanRate")); - assertNotNull(map.toString(), map.get("1minRate")); - assertNotNull(map.toString(), map.get("5minRate")); - assertNotNull(map.toString(), map.get("15minRate")); - assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0); - map.clear(); - - String key2 = "solr\\.core\\..*:.*/select\\.request.*"; - resp = new SolrQueryResponse(); - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.EXPR_PARAM, - key2), + MetricsHandler.PROMETHEUS_METRICS_WT, + MetricsHandler.CATEGORY_PARAM, + "CORE", + MetricsHandler.METRIC_NAME_PARAM, + expectedMetricName), resp); - // response structure is like in the case of non-key params - val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null); - assertNotNull(val); - Object v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requestTimes"); - assertNotNull(v); - assertTrue(v instanceof MapWriter); - ((MapWriter) v).toMap(map); - assertEquals(map.toString(), 14, map.size()); - assertNotNull(map.toString(), map.get("1minRate")); - assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0); - map.clear(); - // select requests counter - v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests"); - assertNotNull(v); - assertTrue(v instanceof Number); - - // test multiple expressions producing overlapping metrics - should be no dupes - - // this key matches also sub-metrics of /select, eg. /select[shard], ... - String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count"; - resp = new SolrQueryResponse(); - // ORDER OF PARAMS MATTERS HERE! see the refguide - handler.handleRequestBody( - req( - CommonParams.QT, - "/admin/metrics", - CommonParams.WT, - "json", - MetricsHandler.EXPR_PARAM, - key2, - MetricsHandler.EXPR_PARAM, - key1, - MetricsHandler.EXPR_PARAM, - key3), - resp); - val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null); - assertNotNull(val); - // for requestTimes only the full set of values from the first expr should be present - assertNotNull(val); - SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val; - assertEquals(values.jsonStr(), 3, values.size()); - v = values.get("QUERY./select.requestTimes"); - assertTrue(v instanceof MapWriter); - ((MapWriter) v).toMap(map); - assertTrue(map.toString(), map.containsKey("count")); - map.clear(); - v = values.get("QUERY./select[shard].requestTimes"); - assertTrue(v instanceof MapWriter); - ((MapWriter) v).toMap(map); - assertTrue(map.toString(), map.containsKey("count")); - } - private MetricSnapshot getMetricSnapshot(MetricSnapshots snapshots, String metricName) { - return snapshots.stream() - .filter(ss -> ss.getMetadata().getPrometheusName().equals(metricName)) - .findAny() - .get(); - } - - private GaugeSnapshot.GaugeDataPointSnapshot getGaugeDatapointSnapshot( - MetricSnapshot snapshot, Labels labels) { - return (GaugeSnapshot.GaugeDataPointSnapshot) - snapshot.getDataPoints().stream() - .filter(ss -> ss.getLabels().hasSameValues(labels)) - .findAny() - .get(); - } - - private CounterSnapshot.CounterDataPointSnapshot getCounterDatapointSnapshot( - MetricSnapshot snapshot, Labels labels) { - return (CounterSnapshot.CounterDataPointSnapshot) - snapshot.getDataPoints().stream() - .filter(ss -> ss.getLabels().hasSameValues(labels)) - .findAny() - .get(); - } - - private SummarySnapshot.SummaryDataPointSnapshot getSummaryDataPointSnapshot( - MetricSnapshot snapshot, Labels labels) { - return (SummarySnapshot.SummaryDataPointSnapshot) - snapshot.getDataPoints().stream() - .filter(ss -> ss.getLabels().hasSameValues(labels)) - .findAny() - .get(); - } - - static class RefreshablePluginHolder extends PluginBag.PluginHolder<SolrRequestHandler> { - - private DumpRequestHandler rh; - private SolrMetricsContext metricsInfo; - - public RefreshablePluginHolder(PluginInfo info, DumpRequestHandler rh) { - super(info); - this.rh = rh; - } - - @Override - public boolean isLoaded() { - return true; - } - - void closeHandler() throws Exception { - this.metricsInfo = rh.getSolrMetricsContext(); - // if(metricsInfo.tag.contains(String.valueOf(rh.hashCode()))){ - // //this created a new child metrics - // metricsInfo = metricsInfo.getParent(); - // } - this.rh.close(); - } - - void reset(DumpRequestHandler rh) { - this.rh = rh; - if (metricsInfo != null) - this.rh.initializeMetrics(metricsInfo, Attributes.empty(), "/dumphandler"); - } - - @Override - public SolrRequestHandler get() { - return rh; - } + var metrics = (MetricSnapshots) resp.getValues().get("metrics"); + assertEquals(1, metrics.size()); + var actualDatapoint = metrics.get(0).getDataPoints().getFirst(); + assertEquals(expectedMetricName, metrics.get(0).getMetadata().getPrometheusName()); + assertEquals("CORE", actualDatapoint.getLabels().get(MetricsHandler.CATEGORY_PARAM)); + handler.close(); } public static class DumpRequestHandler extends RequestHandlerBase { diff --git a/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java new file mode 100644 index 00000000000..ec91e149b4b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel; + +import io.prometheus.metrics.model.snapshots.Labels; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.Test; + +/** + * Test class for FilterablePrometheusMetricReader which focuses on requiredLabelsFilter method + * which filters metric data points based on label requirements. + */ +public class FilterablePrometheusMetricReaderTest extends SolrTestCaseJ4 { + + Labels actualLabels = Labels.of("key1", "value1", "key2", "value2", "key3", "value3"); + + @Test + public void testFilterMatchingLabel() { + SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of("value1"))); + assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterOneMatchingLabelValue() { + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>(Map.of("key1", Set.of("value1", "value123", "value456", "value789"))); + assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterNoMatchingLabel() { + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>(Map.of("dummyKey", Set.of("dummyValue"))); + assertFalse( + FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterAllMultipleMatchingLabels() { + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value2"))); + assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterMultipleWithOneLabelValueNotMatching() { + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value999"))); + assertFalse( + FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterMultipleWithOneLabelMissing() { + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>(Map.of("key1", Set.of("value1"), "key999", Set.of("value2"))); + assertFalse( + FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterLabelsWithMixedValues() { + // Scenario with multiple label values some matching and some not + SortedMap<String, Set<String>> requiredLabels = + new TreeMap<>( + Map.of("key1", Set.of("value1", "value123"), "key2", Set.of("value456", "value2"))); + assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } + + @Test + public void testFilterEmptyLabelValues() { + // Edge case where label has empty set of allowed values + SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of())); + assertFalse( + FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels)); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java index 09aa0489949..ecfd83c02d4 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java @@ -102,7 +102,7 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 { var segmentSize = SolrMetricTestUtils.getGaugeDatapoint( core, - "solr_core_segment_count", + "solr_core_segments", SolrMetricTestUtils.newStandaloneLabelsBuilder(core) .label("category", "CORE") .build());
