This is an automated email from the ASF dual-hosted git repository. mlbiscoc pushed a commit to branch feature/SOLR-17458-rebased in repository https://gitbox.apache.org/repos/asf/solr.git
commit 8e02bfc4e87120e0a4a297b08a863245d803d151 Author: Matthew Biscocho <[email protected]> AuthorDate: Wed Aug 6 10:35:29 2025 -0400 SOLR-17818: Expose Open Metrics 1.0 format from /admin/metrics and exemplar support (#3427) * Add exemplar support * change open metrics to wt=prometheus * Open metrics format based on accept header --- .../src/java/org/apache/solr/core/SolrCore.java | 2 + .../apache/solr/handler/admin/MetricsHandler.java | 7 +- .../org/apache/solr/metrics/SolrMetricManager.java | 11 +-- .../solr/response/PrometheusResponseWriter.java | 34 +++++++- .../response/TestPrometheusResponseWriter.java | 62 ++++++++++++++ .../solr/opentelemetry/TestDistributedTracing.java | 2 +- .../solr/opentelemetry/TestMetricExemplars.java | 95 ++++++++++++++++++++++ 7 files changed, 203 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 825e8b11094..bfe73060d22 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -17,6 +17,7 @@ package org.apache.solr.core; import static org.apache.solr.common.params.CommonParams.PATH; +import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT; import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT; import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR; import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR; @@ -3102,6 +3103,7 @@ public class SolrCore implements SolrInfoBean, Closeable { m.put("schema.xml", new SchemaXmlResponseWriter()); m.put("smile", new SmileResponseWriter()); m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter()); + m.put(OPEN_METRICS_WT, new PrometheusResponseWriter()); m.put(ReplicationAPIBase.FILE_STREAM, getFileStreamWriter()); DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m); try { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java index ba7f6bba34b..f55c0c4b177 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java @@ -69,7 +69,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName public static final String KEY_PARAM = "key"; public static final String EXPR_PARAM = "expr"; public static final String TYPE_PARAM = "type"; + // NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metrics public static final String PROMETHEUS_METRICS_WT = "prometheus"; + public static final String OPEN_METRICS_WT = "openmetrics"; public static final String ALL = "all"; @@ -126,8 +128,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName return; } - // TODO SOLR-17458: Make this the default option after dropwizard removal - if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) { + // NOCOMMIT SOLR-17458: Make this the default option after dropwizard removal + if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT)) + || OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) { consumer.accept("metrics", metricManager.getPrometheusMetricReaders()); return; } diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index 7056a457861..0e6a1db637c 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -52,6 +52,8 @@ import io.opentelemetry.api.metrics.ObservableLongMeasurement; import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; +import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -725,9 +727,7 @@ public class SolrMetricManager { } /** - * Get (or create if not present) a named {@link SdkMeterProvider} under {@link - * MeterProviderAndReaders}. This also registers a corresponding {@link PrometheusMetricReader} - * Dropwizards's {@link SolrMetricManager#registry(String)} equivalent + * Get (or create if not present) a named {@link SdkMeterProvider}. * * @param providerName name of the meter provider and prometheus metric reader * @return existing or newly created meter provider @@ -741,8 +741,9 @@ public class SolrMetricManager { var reader = new PrometheusMetricReader(true, null); // NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP // with an exporter - var provider = SdkMeterProvider.builder().registerMetricReader(reader).build(); - return new MeterProviderAndReaders(provider, reader); + var provider = SdkMeterProvider.builder().registerMetricReader(reader); + SdkMeterProviderUtil.setExemplarFilter(provider, ExemplarFilter.traceBased()); + return new MeterProviderAndReaders(provider.build(), reader); }) .sdkMeterProvider(); } diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java index 77fd3642b2a..070ed662dc0 100644 --- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java @@ -16,8 +16,11 @@ */ package org.apache.solr.response; +import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT; + import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; import io.prometheus.metrics.model.snapshots.CounterSnapshot; import io.prometheus.metrics.model.snapshots.GaugeSnapshot; @@ -31,6 +34,7 @@ import java.lang.invoke.MethodHandles; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.handler.admin.MetricsHandler; import org.apache.solr.request.SolrQueryRequest; import org.slf4j.Logger; @@ -42,6 +46,9 @@ public class PrometheusResponseWriter implements QueryResponseWriter { // not TextQueryResponseWriter because Prometheus libs work with an OutputStream private static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4"; + private static final String CONTENT_TYPE_OPEN_METRICS = + "application/openmetrics-text; version=1.0.0; charset=utf-8"; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Override @@ -55,12 +62,35 @@ public class PrometheusResponseWriter implements QueryResponseWriter { List<MetricSnapshot> snapshots = readers.values().stream().flatMap(r -> r.collect().stream()).toList(); - new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots)); + if (writeOpenMetricsFormat(request)) { + new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots)); + } else { + new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots)); + } } @Override public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return CONTENT_TYPE_PROMETHEUS; + return writeOpenMetricsFormat(request) ? CONTENT_TYPE_OPEN_METRICS : CONTENT_TYPE_PROMETHEUS; + } + + private boolean writeOpenMetricsFormat(SolrQueryRequest request) { + String wt = request.getParams().get(CommonParams.WT); + if (OPEN_METRICS_WT.equals(wt)) { + return true; + } + + String acceptHeader = + request.getHttpSolrCall() != null + ? request.getHttpSolrCall().getReq().getHeader("Accept") + : null; + + if (acceptHeader == null) { + return false; + } + + return acceptHeader.contains("application/openmetrics-text") + && (acceptHeader.contains("version=1.0.0")); } /** diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index 6d818157b6f..71545ea6628 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -17,7 +17,9 @@ package org.apache.solr.response; import com.codahale.metrics.SharedMetricRegistries; +import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -28,6 +30,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.common.params.ModifiableSolrParams; @@ -118,4 +121,63 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { }); } } + + @Test + public void testAcceptHeaderOpenMetricsFormat() throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params); + + // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + req.addHeader("Accept", "application/openmetrics-text;version=1.0.0"); + + try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { + NamedList<Object> res = adminClient.request(req); + + try (InputStream in = (InputStream) res.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + assertTrue( + "Should use OpenMetrics format when Accept header is set", + output.trim().endsWith("# EOF")); + } + } + } + + @Test + public void testWtParameterOpenMetricsFormat() throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params); + req.setResponseParser(new InputStreamResponseParser("openmetrics")); + + try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { + NamedList<Object> res = adminClient.request(req); + + try (InputStream in = (InputStream) res.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + assertTrue( + "Should use OpenMetrics format when Accept header is set", + output.trim().endsWith("# EOF")); + } + } + } + + @Test + public void testDefaultPrometheusFormat() throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params); + // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { + NamedList<Object> res = adminClient.request(req); + + try (InputStream in = (InputStream) res.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + assertFalse( + "Should default to Prometheus format when no Accept header or wt=openmetrics is set", + output.trim().endsWith("# EOF")); + } + } + } } diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java index 501893e99e8..4daba5c87a4 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java @@ -321,7 +321,7 @@ public class TestDistributedTracing extends SolrCloudTestCase { return result; } - private String getRootTraceId(List<SpanData> finishedSpans) { + static String getRootTraceId(List<SpanData> finishedSpans) { assertEquals(1, finishedSpans.stream().filter(TestDistributedTracing::isRootSpan).count()); return finishedSpans.stream() .filter(TestDistributedTracing::isRootSpan) diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java new file mode 100644 index 00000000000..687b86b20cb --- /dev/null +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java @@ -0,0 +1,95 @@ +package org.apache.solr.opentelemetry; + +import static org.apache.solr.opentelemetry.TestDistributedTracing.getAndClearSpans; +import static org.apache.solr.opentelemetry.TestDistributedTracing.getRootTraceId; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.TracerProvider; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestMetricExemplars extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("otel.traces.sampler", "always_on"); + // force early init + CustomTestOtelTracerConfigurator.prepareForTest(); + + configureCluster(1) + .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) + .withSolrXml(TEST_PATH().resolve("solr.xml")) + .withTraceIdGenerationDisabled() + .configure(); + + assertNotEquals( + "Expecting active otel, not noop impl", + TracerProvider.noop(), + GlobalOpenTelemetry.get().getTracerProvider()); + + CollectionAdminRequest.createCollection("collection1", "config", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection("collection1", 1, 1); + } + + @AfterClass + public static void afterClass() { + CustomTestOtelTracerConfigurator.resetForTest(); + } + + @Before + private void resetSpanData() { + getAndClearSpans(); + } + + @Test + public void testOpenMetricExemplars() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + + // Generate exemplars + cloudClient.add("collection1", sdoc("id", "1")); + var spans = getAndClearSpans(); + var expectedTrace = getRootTraceId(spans); + + var req = + new GenericSolrRequest( + SolrRequest.METHOD.GET, + "/admin/metrics", + SolrRequest.SolrRequestType.ADMIN, + new ModifiableSolrParams().set("wt", "openmetrics")); + req.setResponseParser(new InputStreamResponseParser("openmetrics")); + NamedList<Object> resp = cloudClient.request(req); + + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + var line = + output + .lines() + .filter( + l -> + l.startsWith("solr_metrics_core_requests_total") + && l.contains("handler=\"/update\"") + && l.contains("collection=\"collection1\"")) + .findFirst() + .orElseThrow( + () -> + new AssertionError( + "Could not find /update solr_metrics_core_requests_total")); + + String actualExemplarTrace = line.split("trace_id=\"")[1].split("\"")[0]; + assertEquals(actualExemplarTrace, expectedTrace); + } + } +}
