This is an automated email from the ASF dual-hosted git repository. mlbiscoc pushed a commit to branch feature/SOLR-17458-rebased in repository https://gitbox.apache.org/repos/asf/solr.git
commit 50af82fbbc124d78cb57368cc6b2f4cdb09de577 Author: Matthew Biscocho <[email protected]> AuthorDate: Mon Jun 30 12:50:14 2025 -0400 SOLR-17793: Create dedicated SdkMeterProviders for Solr core metrics (#3402) * Create dynamic SDK Meter Providers and tests * Bad merge from feature branch * Wrap meter providers and metric readers * Test cleanup * Cleanup * Fix double lookups * Create a separate test suite for prometheus * Move back to registry name * Changes from comments --- .../java/org/apache/solr/core/CoreContainer.java | 6 +- .../solr/core/OpenTelemetryConfigurator.java | 25 +- .../apache/solr/handler/RequestHandlerBase.java | 2 +- .../apache/solr/handler/admin/MetricsHandler.java | 2 +- .../apache/solr/metrics/SolrCoreMetricManager.java | 1 + .../org/apache/solr/metrics/SolrMetricManager.java | 160 ++++++++++--- .../apache/solr/metrics/SolrMetricsContext.java | 22 +- .../solr/response/PrometheusResponseWriter.java | 112 ++++++++- .../org/apache/solr/util/stats/MetricUtils.java | 25 -- .../apache/solr/core/TestTracerConfigurator.java | 2 +- .../solr/handler/admin/MetricsHandlerTest.java | 1 + .../org/apache/solr/metrics/MetricsConfigTest.java | 10 +- .../apache/solr/metrics/SolrMetricManagerTest.java | 262 +++++++++++++++++++++ .../response/TestPrometheusResponseWriter.java | 46 +--- .../TestPrometheusResponseWriterCloud.java | 164 +++++++++++++ .../src/java/org/apache/solr/SolrTestCaseJ4.java | 4 +- .../apache/solr/cloud/MiniSolrCloudCluster.java | 1 + 17 files changed, 704 insertions(+), 141 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 75e1d43d60b..02b63fe0c16 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -33,7 +33,6 @@ import com.github.benmanes.caffeine.cache.Interner; import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; import jakarta.inject.Singleton; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -404,9 +403,8 @@ public class CoreContainer { this.solrHome = config.getSolrHome(); this.solrCores = SolrCores.newSolrCores(this); this.nodeKeyPair = new SolrNodeKeyPair(cfg.getCloudConfig()); - PrometheusMetricReader metricReader = new PrometheusMetricReader(true, null); - OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader, metricReader); - this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig(), metricReader); + OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader); + this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig()); this.tracer = TraceUtils.getGlobalTracer(); containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(nodeKeyPair)); diff --git a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java index 47547b4950f..1be4b0deb6b 100644 --- a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java +++ b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java @@ -19,16 +19,12 @@ package org.apache.solr.core; import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; import io.opentelemetry.context.propagation.ContextPropagators; import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.OpenTelemetrySdkBuilder; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.metrics.export.MetricReader; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.samplers.Sampler; import java.lang.invoke.MethodHandles; import java.util.Locale; import java.util.Map; @@ -62,7 +58,7 @@ public abstract class OpenTelemetryConfigurator implements NamedListInitializedP * SDK. */ public static synchronized void initializeOpenTelemetrySdk( - NodeConfig cfg, SolrResourceLoader loader, MetricReader metricReader) { + NodeConfig cfg, SolrResourceLoader loader) { PluginInfo info = (cfg != null) ? cfg.getTracerConfiguratorPluginInfo() : null; if (info != null && info.isEnabled()) { @@ -71,28 +67,21 @@ public abstract class OpenTelemetryConfigurator implements NamedListInitializedP } else if (OpenTelemetryConfigurator.shouldAutoConfigOTEL()) { OpenTelemetryConfigurator.autoConfigureOpenTelemetrySdk(loader); } else { - // Initializing sampler as always off to replicate no-op Tracer provider - OpenTelemetryConfigurator.configureOpenTelemetrySdk( - SdkMeterProvider.builder().registerMetricReader(metricReader).build(), - SdkTracerProvider.builder().setSampler(Sampler.alwaysOff()).build()); + OpenTelemetryConfigurator.configureOpenTelemetrySdk(); } } - private static void configureOpenTelemetrySdk( - SdkMeterProvider sdkMeterProvider, SdkTracerProvider sdkTracerProvider) { + private static void configureOpenTelemetrySdk() { if (loaded) return; - OpenTelemetrySdkBuilder builder = OpenTelemetrySdk.builder(); if (TRACE_ID_GEN_ENABLED) { log.info("OpenTelemetry tracer enabled with simple propagation only."); ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider()); - builder.setPropagators(ContextPropagators.create(SimplePropagator.getInstance())); } - if (sdkMeterProvider != null) builder.setMeterProvider(sdkMeterProvider); - if (sdkTracerProvider != null) builder.setTracerProvider(sdkTracerProvider); - - GlobalOpenTelemetry.set(builder.build()); + OpenTelemetry otel = + OpenTelemetry.propagating(ContextPropagators.create(SimplePropagator.getInstance())); + GlobalOpenTelemetry.set(otel); loaded = true; } diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index e633a91113d..6038db647a3 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -192,7 +192,7 @@ public abstract class RequestHandlerBase new HandlerMetrics( new SolrMetricsContext( new SolrMetricManager( - null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build(), null), + null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build()), "NO_OP", "NO_OP"), Attributes.empty()); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java index f2b501b8a41..ba7f6bba34b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java @@ -128,7 +128,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName // TODO SOLR-17458: Make this the default option after dropwizard removal if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) { - consumer.accept("metrics", metricManager.getMetricReader().collect()); + consumer.accept("metrics", metricManager.getPrometheusMetricReaders()); return; } diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java index 62a342180f2..9d79ada8a85 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java @@ -108,6 +108,7 @@ public class SolrCoreMetricManager implements Closeable { * and will be used under the new core name. This method also reloads reporters so that they use * the new core name. */ + // NOCOMMIT SOLR-17458: Update for core renaming public void afterCoreRename() { assert core.getCoreDescriptor().getCloudDescriptor() == null; String oldRegistryName = solrMetricsContext.getRegistryName(); diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index bcefc0b3ddc..7056a457861 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -42,8 +42,8 @@ import io.opentelemetry.api.metrics.LongHistogram; import io.opentelemetry.api.metrics.LongHistogramBuilder; import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; -import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.api.metrics.ObservableDoubleCounter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter; import io.opentelemetry.api.metrics.ObservableLongCounter; @@ -51,6 +51,7 @@ import io.opentelemetry.api.metrics.ObservableLongGauge; import io.opentelemetry.api.metrics.ObservableLongMeasurement; import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -81,7 +82,6 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.logging.MDCLoggingContext; -import org.apache.solr.util.stats.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -115,6 +115,8 @@ public class SolrMetricManager { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String OTEL_SCOPE_NAME = "org.apache.solr"; + /** Common prefix for all registry names that Solr uses. */ public static final String REGISTRY_NAME_PREFIX = "solr."; @@ -147,9 +149,8 @@ public class SolrMetricManager { private final MetricRegistry.MetricSupplier<Timer> timerSupplier; private final MetricRegistry.MetricSupplier<Histogram> histogramSupplier; - private final MeterProvider meterProvider; - private final Map<String, io.opentelemetry.api.metrics.Meter> meters = new ConcurrentHashMap<>(); - private final PrometheusMetricReader metricReader; + private final ConcurrentMap<String, MeterProviderAndReaders> meterProviderAndReaders = + new ConcurrentHashMap<>(); public SolrMetricManager() { metricsConfig = new MetricsConfig.MetricsConfigBuilder().build(); @@ -157,15 +158,10 @@ public class SolrMetricManager { meterSupplier = MetricSuppliers.meterSupplier(null, null); timerSupplier = MetricSuppliers.timerSupplier(null, null); histogramSupplier = MetricSuppliers.histogramSupplier(null, null); - meterProvider = MetricUtils.getMeterProvider(); - metricReader = null; } - public SolrMetricManager( - SolrResourceLoader loader, MetricsConfig metricsConfig, PrometheusMetricReader metricReader) { + public SolrMetricManager(SolrResourceLoader loader, MetricsConfig metricsConfig) { this.metricsConfig = metricsConfig; - this.metricReader = metricReader; - this.meterProvider = MetricUtils.getMeterProvider(); counterSupplier = MetricSuppliers.counterSupplier(loader, metricsConfig.getCounterSupplier()); meterSupplier = MetricSuppliers.meterSupplier(loader, metricsConfig.getMeterSupplier()); timerSupplier = MetricSuppliers.timerSupplier(loader, metricsConfig.getTimerSupplier()); @@ -176,7 +172,10 @@ public class SolrMetricManager { public LongCounter longCounter( String registry, String counterName, String description, String unit) { LongCounterBuilder builder = - meterProvider.get(registry).counterBuilder(counterName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .counterBuilder(counterName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.build(); @@ -185,7 +184,10 @@ public class SolrMetricManager { public LongUpDownCounter longUpDownCounter( String registry, String counterName, String description, String unit) { LongUpDownCounterBuilder builder = - meterProvider.get(registry).upDownCounterBuilder(counterName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .upDownCounterBuilder(counterName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.build(); @@ -194,8 +196,8 @@ public class SolrMetricManager { public DoubleUpDownCounter doubleUpDownCounter( String registry, String counterName, String description, String unit) { DoubleUpDownCounterBuilder builder = - meterProvider - .get(registry) + meterProvider(registry) + .get(OTEL_SCOPE_NAME) .upDownCounterBuilder(counterName) .setDescription(description) .ofDoubles(); @@ -207,8 +209,8 @@ public class SolrMetricManager { public DoubleCounter doubleCounter( String registry, String counterName, String description, String unit) { DoubleCounterBuilder builder = - meterProvider - .get(registry) + meterProvider(registry) + .get(OTEL_SCOPE_NAME) .counterBuilder(counterName) .setDescription(description) .ofDoubles(); @@ -220,7 +222,10 @@ public class SolrMetricManager { public DoubleHistogram doubleHistogram( String registry, String histogramName, String description, String unit) { DoubleHistogramBuilder builder = - meterProvider.get(registry).histogramBuilder(histogramName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .histogramBuilder(histogramName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.build(); @@ -229,8 +234,8 @@ public class SolrMetricManager { public LongHistogram longHistogram( String registry, String histogramName, String description, String unit) { LongHistogramBuilder builder = - meterProvider - .get(registry) + meterProvider(registry) + .get(OTEL_SCOPE_NAME) .histogramBuilder(histogramName) .setDescription(description) .ofLongs(); @@ -243,7 +248,10 @@ public class SolrMetricManager { public DoubleGauge doubleGauge( String registry, String gaugeName, String description, String unit) { DoubleGaugeBuilder builder = - meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .gaugeBuilder(gaugeName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.build(); @@ -251,7 +259,11 @@ public class SolrMetricManager { public LongGauge longGauge(String registry, String gaugeName, String description, String unit) { LongGaugeBuilder builder = - meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description).ofLongs(); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .gaugeBuilder(gaugeName) + .setDescription(description) + .ofLongs(); if (unit != null) builder.setUnit(unit); return builder.build(); @@ -264,7 +276,10 @@ public class SolrMetricManager { Consumer<ObservableLongMeasurement> callback, String unit) { LongCounterBuilder builder = - meterProvider.get(registry).counterBuilder(counterName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .counterBuilder(counterName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.buildWithCallback(callback); @@ -277,8 +292,8 @@ public class SolrMetricManager { Consumer<ObservableDoubleMeasurement> callback, String unit) { DoubleCounterBuilder builder = - meterProvider - .get(registry) + meterProvider(registry) + .get(OTEL_SCOPE_NAME) .counterBuilder(counterName) .setDescription(description) .ofDoubles(); @@ -294,7 +309,28 @@ public class SolrMetricManager { Consumer<ObservableLongMeasurement> callback, String unit) { LongGaugeBuilder builder = - meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description).ofLongs(); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .gaugeBuilder(gaugeName) + .setDescription(description) + .ofLongs(); + if (unit != null) builder.setUnit(unit); + + return builder.buildWithCallback(callback); + } + + public ObservableDoubleGauge observableDoubleGauge( + String registry, + String gaugeName, + String description, + Consumer<ObservableDoubleMeasurement> callback, + String unit) { + DoubleGaugeBuilder builder = + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .gaugeBuilder(gaugeName) + .setDescription(description); + if (unit != null) builder.setUnit(unit); return builder.buildWithCallback(callback); @@ -307,7 +343,10 @@ public class SolrMetricManager { Consumer<ObservableLongMeasurement> callback, String unit) { LongUpDownCounterBuilder builder = - meterProvider.get(registry).upDownCounterBuilder(counterName).setDescription(description); + meterProvider(registry) + .get(OTEL_SCOPE_NAME) + .upDownCounterBuilder(counterName) + .setDescription(description); if (unit != null) builder.setUnit(unit); return builder.buildWithCallback(callback); @@ -320,8 +359,8 @@ public class SolrMetricManager { Consumer<ObservableDoubleMeasurement> callback, String unit) { DoubleUpDownCounterBuilder builder = - meterProvider - .get(registry) + meterProvider(registry) + .get(OTEL_SCOPE_NAME) .upDownCounterBuilder(counterName) .setDescription(description) .ofDoubles(); @@ -330,10 +369,6 @@ public class SolrMetricManager { return builder.buildWithCallback(callback); } - public io.opentelemetry.api.metrics.Meter getOtelRegistry(String registryName) { - return meters.get(registryName); - } - // for unit tests public MetricRegistry.MetricSupplier<Counter> getCounterSupplier() { return counterSupplier; @@ -578,6 +613,7 @@ public class SolrMetricManager { } /** Return a set of existing registry names. */ + // NOCOMMIT: Remove for OTEL public Set<String> registryNames() { Set<String> set = new HashSet<>(); set.addAll(registries.keySet()); @@ -598,6 +634,10 @@ public class SolrMetricManager { return names.contains(name); } + public boolean hasMeterProvider(String name) { + return meterProviderAndReaders.containsKey(enforcePrefix(name)); + } + /** * Return set of existing registry names that match a regex pattern * @@ -654,7 +694,7 @@ public class SolrMetricManager { * @param registry name of the registry * @return existing or newly created registry */ - // TODO SOLR-17458: We may not need + // NOCOMMIT SOLR-17458: We may not need public MetricRegistry registry(String registry) { return registry(registry, true); } @@ -684,6 +724,29 @@ public class SolrMetricManager { } } + /** + * Get (or create if not present) a named {@link SdkMeterProvider} under {@link + * MeterProviderAndReaders}. This also registers a corresponding {@link PrometheusMetricReader} + * Dropwizards's {@link SolrMetricManager#registry(String)} equivalent + * + * @param providerName name of the meter provider and prometheus metric reader + * @return existing or newly created meter provider + */ + public SdkMeterProvider meterProvider(String providerName) { + providerName = enforcePrefix(providerName); + return meterProviderAndReaders + .computeIfAbsent( + providerName, + key -> { + var reader = new PrometheusMetricReader(true, null); + // NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP + // with an exporter + var provider = SdkMeterProvider.builder().registerMetricReader(reader).build(); + return new MeterProviderAndReaders(provider, reader); + }) + .sdkMeterProvider(); + } + // TODO SOLR-17458: We may not need private static MetricRegistry getOrCreateRegistry( ConcurrentMap<String, MetricRegistry> map, String registry) { @@ -702,12 +765,14 @@ public class SolrMetricManager { } /** - * Remove a named registry. + * Remove a named registry and close an existing {@link SdkMeterProvider}. Upon closing of + * provider, all metric readers registered to it are closed. * * @param registry name of the registry to remove */ // TODO SOLR-17458: You can't delete OTEL meters public void removeRegistry(String registry) { + // NOCOMMIT Remove all closing Dropwizard registries // close any reporters for this registry first closeReporters(registry, null); // make sure we use a name with prefix @@ -722,6 +787,12 @@ public class SolrMetricManager { swapLock.unlock(); } } + meterProviderAndReaders.computeIfPresent( + registry, + (key, meterAndReader) -> { + meterAndReader.sdkMeterProvider().close(); + return null; + }); } /** @@ -827,8 +898,9 @@ public class SolrMetricManager { * start with the prefix will be removed. * @return set of metrics names that have been removed. */ - // TODO SOLR-17458: This is not supported in otel. Metrics are immutable. We can at best filter - // them + // NOCOMMIT SOLR-17458: This is not supported in otel. Metrics are immutable. We can at best + // filter + // them or delete the meterProvider entirely public Set<String> clearMetrics(String registry, String... metricPath) { PrefixFilter filter; if (metricPath == null || metricPath.length == 0) { @@ -1603,7 +1675,17 @@ public class SolrMetricManager { return metricsConfig; } - public PrometheusMetricReader getMetricReader() { - return this.metricReader; + /** Get a shallow copied map of {@link PrometheusMetricReader}. */ + public Map<String, PrometheusMetricReader> getPrometheusMetricReaders() { + return meterProviderAndReaders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().prometheusMetricReader())); + } + + public PrometheusMetricReader getPrometheusMetricReader(String providerName) { + MeterProviderAndReaders mpr = meterProviderAndReaders.get(enforcePrefix(providerName)); + return (mpr != null) ? mpr.prometheusMetricReader() : null; } + + private record MeterProviderAndReaders( + SdkMeterProvider sdkMeterProvider, PrometheusMetricReader prometheusMetricReader) {} } diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java index 269575798ec..222ce2e0e8a 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java @@ -30,6 +30,8 @@ import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; import io.opentelemetry.api.metrics.LongHistogram; import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; import io.opentelemetry.api.metrics.ObservableLongGauge; import io.opentelemetry.api.metrics.ObservableLongMeasurement; import java.util.Map; @@ -206,17 +208,31 @@ public class SolrMetricsContext { public ObservableLongGauge observableLongGauge( String metricName, String description, Consumer<ObservableLongMeasurement> callback) { - return metricManager.observableLongGauge(registryName, metricName, description, callback, null); + return observableLongGauge(metricName, description, callback, null); } public ObservableLongGauge observableLongGauge( String metricName, String description, - String unit, - Consumer<ObservableLongMeasurement> callback) { + Consumer<ObservableLongMeasurement> callback, + String unit) { return metricManager.observableLongGauge(registryName, metricName, description, callback, unit); } + public ObservableDoubleGauge observableDoubleGauge( + String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) { + return observableDoubleGauge(metricName, description, callback, null); + } + + public ObservableDoubleGauge observableDoubleGauge( + String metricName, + String description, + Consumer<ObservableDoubleMeasurement> callback, + String unit) { + return metricManager.observableDoubleGauge( + registryName, metricName, description, callback, unit); + } + /** * Convenience method for {@link SolrMetricManager#meter(SolrMetricsContext, String, String, * String...)}. diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java index 5a375bdb408..77fd3642b2a 100644 --- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java @@ -16,19 +16,27 @@ */ package org.apache.solr.response; +import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.InfoSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.io.IOException; import java.io.OutputStream; import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.solr.handler.admin.MetricsHandler; import org.apache.solr.request.SolrQueryRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Response writer for Prometheus metrics. This is used only by the {@link - * org.apache.solr.handler.admin.MetricsHandler} - */ +/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */ @SuppressWarnings(value = "unchecked") public class PrometheusResponseWriter implements QueryResponseWriter { // not TextQueryResponseWriter because Prometheus libs work with an OutputStream @@ -40,12 +48,104 @@ public class PrometheusResponseWriter implements QueryResponseWriter { public void write( OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType) throws IOException { - var prometheusTextFormatWriter = new PrometheusTextFormatWriter(false); - prometheusTextFormatWriter.write(out, (MetricSnapshots) response.getValues().get("metrics")); + + Map<String, PrometheusMetricReader> readers = + (Map<String, PrometheusMetricReader>) response.getValues().get("metrics"); + + List<MetricSnapshot> snapshots = + readers.values().stream().flatMap(r -> r.collect().stream()).toList(); + + new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots)); } @Override public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { return CONTENT_TYPE_PROMETHEUS; } + + /** + * Merge a collection of individual {@link MetricSnapshot} instances into one {@link + * MetricSnapshots}. This is necessary because we create a {@link SdkMeterProvider} per Solr core + * resulting in duplicate metric names across cores which is an illegal format if not under the + * same prometheus grouping. + */ + private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) { + Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>(); + Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>(); + Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>(); + InfoSnapshot otelInfoSnapshots = null; + + for (MetricSnapshot snapshot : snapshots) { + String metricName = snapshot.getMetadata().getPrometheusName(); + + switch (snapshot) { + case CounterSnapshot counterSnapshot -> { + CounterSnapshot.Builder builder = + counterSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + CounterSnapshot.builder() + .name(counterSnapshot.getMetadata().getName()) + .help(counterSnapshot.getMetadata().getHelp()); + return counterSnapshot.getMetadata().hasUnit() + ? base.unit(counterSnapshot.getMetadata().getUnit()) + : base; + }); + counterSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case GaugeSnapshot gaugeSnapshot -> { + GaugeSnapshot.Builder builder = + gaugeSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + GaugeSnapshot.builder() + .name(gaugeSnapshot.getMetadata().getName()) + .help(gaugeSnapshot.getMetadata().getHelp()); + return gaugeSnapshot.getMetadata().hasUnit() + ? base.unit(gaugeSnapshot.getMetadata().getUnit()) + : base; + }); + gaugeSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case HistogramSnapshot histogramSnapshot -> { + HistogramSnapshot.Builder builder = + histogramSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + HistogramSnapshot.builder() + .name(histogramSnapshot.getMetadata().getName()) + .help(histogramSnapshot.getMetadata().getHelp()); + return histogramSnapshot.getMetadata().hasUnit() + ? base.unit(histogramSnapshot.getMetadata().getUnit()) + : base; + }); + histogramSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case InfoSnapshot infoSnapshot -> { + // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate + // metric called target_info containing OTEL SDK metadata. Only one of these need to be + // kept + if (otelInfoSnapshots == null) + otelInfoSnapshots = + new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints()); + } + default -> { + log.warn( + "Unexpected snapshot type: {} for metric {}", + snapshot.getClass().getName(), + snapshot.getMetadata().getName()); + } + } + } + + MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder(); + counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots); + return snapshotsBuilder.build(); + } } diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java index 8b79fc26b35..9cff065c06f 100644 --- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java +++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java @@ -26,10 +26,6 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.MeterProvider; import java.beans.BeanInfo; import java.beans.IntrospectionException; import java.beans.Introspector; @@ -55,7 +51,6 @@ import java.util.function.Predicate; import org.apache.solr.common.ConditionalKeyMapWriter; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.MapWriter; -import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrInfoBean; @@ -857,24 +852,4 @@ public class MetricUtils { } } } - - public static MeterProvider getMeterProvider() { - return GlobalOpenTelemetry.getMeterProvider(); - } - - public static Attributes createAttributes(String... attributes) { - if (attributes.length % 2 == 1) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Odd number of field/value strings passed"); - } - - AttributesBuilder builder = Attributes.builder(); - for (int i = 0; i < attributes.length; i += 2) { - String key = attributes[i]; - String value = attributes[i + 1]; - builder.put(key, value); - } - - return builder.build(); - } } diff --git a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java index ba453efc5d4..5aa8ffec1ed 100644 --- a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java +++ b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java @@ -41,7 +41,7 @@ public class TestTracerConfigurator extends SolrTestCaseJ4 { public void configuratorClassDoesNotExistTest() { assertTrue(OpenTelemetryConfigurator.shouldAutoConfigOTEL()); SolrResourceLoader loader = new SolrResourceLoader(TEST_PATH().resolve("collection1")); - OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, loader, null); + OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, loader); assertEquals( "Expecting noop otel after failure to auto-init", TracerProvider.noop().get(null), diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java index 96001e15b63..07f71176591 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java @@ -48,6 +48,7 @@ import org.junit.BeforeClass; import org.junit.Test; /** Test for {@link MetricsHandler} */ +// NOCOMMIT SOLR-17785: Lets move this to SolrCloudTestCase public class MetricsHandlerTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java b/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java index 32245ffb4c6..eff63f4db72 100644 --- a/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java @@ -35,7 +35,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 { public void testDefaults() { NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml"); SolrMetricManager mgr = - new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null); + new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig()); assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.DefaultCounterSupplier); assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.DefaultMeterSupplier); assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.DefaultTimerSupplier); @@ -54,7 +54,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 { System.setProperty("histogram.reservoir", SlidingTimeWindowReservoir.class.getName()); NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml"); SolrMetricManager mgr = - new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null); + new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig()); assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.DefaultCounterSupplier); assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.DefaultMeterSupplier); assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.DefaultTimerSupplier); @@ -73,7 +73,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 { System.setProperty("histogram.class", MockHistogramSupplier.class.getName()); NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml"); SolrMetricManager mgr = - new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null); + new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig()); assertTrue(mgr.getCounterSupplier() instanceof MockCounterSupplier); assertTrue(mgr.getMeterSupplier() instanceof MockMeterSupplier); assertTrue(mgr.getTimerSupplier() instanceof MockTimerSupplier); @@ -100,7 +100,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 { System.setProperty("metricsEnabled", "false"); NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml"); SolrMetricManager mgr = - new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null); + new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig()); assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.NoOpCounterSupplier); assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.NoOpMeterSupplier); assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.NoOpTimerSupplier); @@ -111,7 +111,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 { public void testMissingValuesConfig() { NodeConfig cfg = loadNodeConfig("solr-metricsconfig1.xml"); SolrMetricManager mgr = - new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null); + new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig()); assertNull("nullNumber", mgr.nullNumber()); assertEquals("notANumber", -1, mgr.notANumber()); assertEquals("nullNumber", "", mgr.nullString()); diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java index 730fe0989aa..2c8f51aec6d 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java @@ -20,10 +20,28 @@ package org.apache.solr.metrics; import com.codahale.metrics.Counter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; +import com.google.common.util.concurrent.AtomicDouble; +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongGauge; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.util.HashMap; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.DoubleAdder; +import java.util.concurrent.atomic.LongAdder; import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.NamedList; @@ -31,10 +49,24 @@ import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.metrics.reporters.MockMetricReporter; +import org.junit.Before; import org.junit.Test; public class SolrMetricManagerTest extends SolrTestCaseJ4 { + final String METER_PROVIDER_NAME = "test_provider_name"; + private SolrMetricManager metricManager; + private PrometheusMetricReader reader; + + @Before + public void setUp() throws Exception { + super.setUp(); + this.metricManager = new SolrMetricManager(); + // Initialize a metric reader for tests + metricManager.meterProvider(METER_PROVIDER_NAME); + this.reader = metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME); + } + // NOCOMMIT: We might not be supported core swapping in 10. Maybe remove this test @Test public void testSwapRegistries() { Random r = random(); @@ -76,6 +108,8 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 { } } + // NOCOMMIT: Migration of this to OTEL isn't possible. You can't register instruments to a + // meterprovider that the provider itself didn't create @Test public void testRegisterAll() throws Exception { Random r = random(); @@ -104,6 +138,8 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 { registryName, mr, SolrMetricManager.ResolutionStrategy.ERROR)); } + // NOCOMMIT: Migration of this to OTEL isn't possible. You can only delete the whole + // sdkMeterProvider and all it's recorded metrics @Test public void testClearMetrics() { Random r = random(); @@ -270,4 +306,230 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 { initArgs.add("configurable", "true"); return new PluginInfo("SolrMetricReporter", attrs, initArgs, null); } + + @Test + public void testLongCounter() { + LongCounter counter = + metricManager.longCounter(METER_PROVIDER_NAME, "my_counter", "desc", null); + counter.add(5); + counter.add(3); + CounterSnapshot actual = snapshot("my_counter", CounterSnapshot.class); + assertEquals(8.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testLongUpDownCounter() { + LongUpDownCounter counter = + metricManager.longUpDownCounter(METER_PROVIDER_NAME, "long_updown_counter", "desc", null); + counter.add(10); + counter.add(-4); + GaugeSnapshot actual = snapshot("long_updown_counter", GaugeSnapshot.class); + assertEquals(6.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testDoubleUpDownCounter() { + DoubleUpDownCounter counter = + metricManager.doubleUpDownCounter( + METER_PROVIDER_NAME, "double_updown_counter", "desc", null); + counter.add(10.0); + counter.add(-5.5); + + GaugeSnapshot actual = snapshot("double_updown_counter", GaugeSnapshot.class); + assertEquals(4.5, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testDoubleCounter() { + DoubleCounter counter = + metricManager.doubleCounter(METER_PROVIDER_NAME, "double_counter", "desc", null); + counter.add(10.0); + counter.add(5.5); + + CounterSnapshot actual = snapshot("double_counter", CounterSnapshot.class); + assertEquals(15.5, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testDoubleHistogram() { + DoubleHistogram histogram = + metricManager.doubleHistogram(METER_PROVIDER_NAME, "double_histogram", "desc", null); + histogram.record(1.1); + histogram.record(2.2); + histogram.record(3.3); + + HistogramSnapshot actual = snapshot("double_histogram", HistogramSnapshot.class); + assertEquals(3, actual.getDataPoints().getFirst().getCount()); + assertEquals(6.6, actual.getDataPoints().getFirst().getSum(), 0.0); + } + + @Test + public void testLongHistogram() { + LongHistogram histogram = + metricManager.longHistogram(METER_PROVIDER_NAME, "long_histogram", "desc", null); + histogram.record(1); + histogram.record(2); + histogram.record(3); + + HistogramSnapshot actual = snapshot("long_histogram", HistogramSnapshot.class); + assertEquals(3, actual.getDataPoints().getFirst().getCount()); + assertEquals(6.0, actual.getDataPoints().getFirst().getSum(), 0.0); + } + + @Test + public void testDoubleGauge() { + DoubleGauge gauge = + metricManager.doubleGauge(METER_PROVIDER_NAME, "double_gauge", "desc", null); + gauge.set(10.0); + gauge.set(5.5); + + GaugeSnapshot actual = snapshot("double_gauge", GaugeSnapshot.class); + assertEquals(5.5, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testLongGauge() { + LongGauge gauge = metricManager.longGauge(METER_PROVIDER_NAME, "long_gauge", "desc", null); + gauge.set(10); + gauge.set(5); + + GaugeSnapshot actual = snapshot("long_gauge", GaugeSnapshot.class); + assertEquals(5.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testObservableLongCounter() { + LongAdder val = new LongAdder(); + metricManager.observableLongCounter( + METER_PROVIDER_NAME, "obs_long_counter", "desc", m -> m.record(val.longValue()), null); + val.add(10); + + CounterSnapshot actual = snapshot("obs_long_counter", CounterSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.add(20); + actual = snapshot("obs_long_counter", CounterSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(30.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testObservableDoubleCounter() { + DoubleAdder val = new DoubleAdder(); + metricManager.observableDoubleCounter( + METER_PROVIDER_NAME, "obs_double_counter", "desc", m -> m.record(val.doubleValue()), null); + val.add(10.0); + + CounterSnapshot actual = snapshot("obs_double_counter", CounterSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.add(0.1); + actual = snapshot("obs_double_counter", CounterSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(10.1, actual.getDataPoints().getFirst().getValue(), 1e-6); + } + + @Test + public void testObservableLongGauge() { + AtomicLong val = new AtomicLong(); + metricManager.observableLongGauge( + METER_PROVIDER_NAME, "obs_long_gauge", "desc", m -> m.record(val.get()), null); + val.set(10L); + + GaugeSnapshot actual = snapshot("obs_long_gauge", GaugeSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.set(20L); + actual = snapshot("obs_long_gauge", GaugeSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(20.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testObservableDoubleGauge() { + AtomicDouble val = new AtomicDouble(); + metricManager.observableDoubleGauge( + METER_PROVIDER_NAME, "obs_double_gauge", "desc", m -> m.record(val.get()), null); + val.set(10.0); + + GaugeSnapshot actual = snapshot("obs_double_gauge", GaugeSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.set(10.1); + actual = snapshot("obs_double_gauge", GaugeSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(10.1, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testObservableLongUpDownCounter() { + LongAdder val = new LongAdder(); + metricManager.observableLongUpDownCounter( + METER_PROVIDER_NAME, "obs_long_updown_gauge", "desc", m -> m.record(val.longValue()), null); + val.add(10L); + + GaugeSnapshot actual = snapshot("obs_long_updown_gauge", GaugeSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.add(-20L); + actual = snapshot("obs_long_updown_gauge", GaugeSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(-10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + } + + @Test + public void testObservableDoubleUpDownCounter() { + DoubleAdder val = new DoubleAdder(); + metricManager.observableDoubleUpDownCounter( + METER_PROVIDER_NAME, + "obs_double_updown_gauge", + "desc", + m -> m.record(val.doubleValue()), + null); + val.add(10.0); + GaugeSnapshot actual = snapshot("obs_double_updown_gauge", GaugeSnapshot.class); + assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0); + + val.add(-20.1); + actual = snapshot("obs_double_updown_gauge", GaugeSnapshot.class); + + // Observable metrics value changes anytime metricReader collects() to trigger callback + assertEquals(-10.1, actual.getDataPoints().getFirst().getValue(), 1e-6); + } + + @Test + public void testCloseMeterProviders() { + LongCounter counter = + metricManager.longCounter(METER_PROVIDER_NAME, "my_counter", "desc", null); + counter.add(5); + + PrometheusMetricReader reader = metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME); + MetricSnapshots metrics = reader.collect(); + CounterSnapshot data = + metrics.stream() + .filter(m -> m.getMetadata().getPrometheusName().equals("my_counter")) + .map(CounterSnapshot.class::cast) + .findFirst() + .orElseThrow(() -> new AssertionError("LongCounter metric not found")); + assertEquals(5, data.getDataPoints().getFirst().getValue(), 0.0); + + metricManager.removeRegistry(METER_PROVIDER_NAME); + + assertNull(metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME)); + } + + // Helper to grab any snapshot by name and type + private <T extends MetricSnapshot> T snapshot(String name, Class<T> cls) { + return reader.collect().stream() + .filter(m -> m.getMetadata().getPrometheusName().equals(name)) + .filter(cls::isInstance) + .map(cls::cast) + .findFirst() + .orElseThrow(() -> new AssertionError("MetricSnapshot not found: " + name)); + } } diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index 4792be1021c..a0010476393 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -16,10 +16,6 @@ */ package org.apache.solr.response; -import com.codahale.metrics.Counter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Meter; -import com.codahale.metrics.SettableGauge; import com.codahale.metrics.SharedMetricRegistries; import java.lang.invoke.MethodHandles; import java.util.Arrays; @@ -36,10 +32,8 @@ import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.SolrJettyTestRule; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -57,25 +51,22 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { solrClientTestRule.startSolr(LuceneTestCase.createTempDir()); solrClientTestRule - .newCollection() + .newCollection("core1") + .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET.toString()) + .create(); + solrClientTestRule + .newCollection("core2") .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET.toString()) .create(); var cc = solrClientTestRule.getCoreContainer(); cc.waitForLoadingCoresToFinish(30000); - SolrMetricManager manager = cc.getMetricManager(); - Counter c = manager.counter(null, "solr.core.collection1", "QUERY./dummy/metrics.requests"); - c.inc(10); - c = manager.counter(null, "solr.node", "ADMIN./dummy/metrics.requests"); - c.inc(20); - Meter m = manager.meter(null, "solr.jetty", "dummyMetrics.2xx-responses"); - m.mark(30); - registerGauge(manager, "solr.jvm", "gc.dummyMetrics.count"); - } + // Populate request metrics on both cores + ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "*:*"); - @AfterClass - public static void clearMetricsRegistries() { - SharedMetricRegistries.clear(); + solrClientTestRule.getSolrClient("core1").query(queryParams); + solrClientTestRule.getSolrClient("core2").query(queryParams); } @Test @@ -87,7 +78,6 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { NamedList<Object> res = adminClient.request(req); - assertNotNull("null response from server", res); String output = (String) res.get("response"); Set<String> seenTypeInfo = new HashSet<>(); @@ -130,20 +120,4 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { }); } } - - private static void registerGauge( - SolrMetricManager metricManager, String registry, String metricName) { - Gauge<Number> metric = - new SettableGauge<>() { - @Override - public void setValue(Number value) {} - - @Override - public Number getValue() { - return 0; - } - }; - metricManager.registerGauge( - null, registry, metric, "", SolrMetricManager.ResolutionStrategy.IGNORE, metricName, ""); - } } diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java new file mode 100644 index 00000000000..4ed29e43199 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestPrometheusResponseWriterCloud extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("metricsEnabled", "true"); + configureCluster(1) + .addConfig( + "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + } + + @Before + public void ensureCollectionsExist() throws Exception { + SolrClient client = cluster.getSolrClient(); + try { + CollectionAdminRequest.deleteCollection("collection1").process(client); + } catch (Exception ignored) { + } + try { + CollectionAdminRequest.deleteCollection("collection2").process(client); + } catch (Exception ignored) { + } + + CollectionAdminRequest.createCollection("collection1", "config", 1, 1).process(client); + CollectionAdminRequest.createCollection("collection2", "config", 1, 1).process(client); + cluster.waitForActiveCollection("collection1", 1, 1); + cluster.waitForActiveCollection("collection2", 1, 1); + } + + @Test + public void testPrometheusCloudLabels() throws Exception { + var solrClient = cluster.getSolrClient(); + + // Increment solr_metrics_core_requests metric for /select + SolrQuery query = new SolrQuery("*:*"); + solrClient.query("collection1", query); + + var req = + new GenericSolrRequest( + METHOD.GET, + "/admin/metrics", + SolrRequestType.ADMIN, + new ModifiableSolrParams().set("wt", "prometheus")); + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + NamedList<Object> resp = solrClient.request(req); + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + assertTrue( + "Missing expected Solr cloud mode prometheus metric with cloud labels", + output + .lines() + .anyMatch( + line -> + line.startsWith("solr_metrics_core_requests_total") + && line.contains("handler=\"/select\"") + && line.contains("collection=\"collection1\"") + && line.contains("core=\"collection1_shard1_replica_n1\"") + && line.contains("replica=\"replica_n1\"") + && line.contains("shard=\"shard1\"") + && line.contains("type=\"requests\""))); + } + } + + @Test + public void testCollectionDeletePrometheusOutput() throws Exception { + var solrClient = cluster.getSolrClient(); + + // Increment solr_metrics_core_requests metric for /select and assert it exists + SolrQuery query = new SolrQuery("*:*"); + solrClient.query("collection1", query); + solrClient.query("collection2", query); + + var req = + new GenericSolrRequest( + METHOD.GET, + "/admin/metrics", + SolrRequestType.ADMIN, + new ModifiableSolrParams().set("wt", "prometheus")); + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + NamedList<Object> resp = solrClient.request(req); + + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + + assertTrue( + "Prometheus output should contains solr_metrics_core_requests for collection1", + output + .lines() + .anyMatch( + line -> + line.startsWith("solr_metrics_core_requests") + && line.contains("collection1"))); + assertTrue( + "Prometheus output should contains solr_metrics_core_requests for collection2", + output + .lines() + .anyMatch( + line -> + line.startsWith("solr_metrics_core_requests") + && line.contains("collection2"))); + } + + // Delete collection and assert metrics have been removed + var deleteRequest = CollectionAdminRequest.deleteCollection("collection1"); + deleteRequest.process(solrClient); + + resp = solrClient.request(req); + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + assertFalse( + "Prometheus output should not contain solr_metrics_core_requests after collection was deleted", + output + .lines() + .anyMatch( + line -> + line.startsWith("solr_metrics_core_requests") + && line.contains("collection1"))); + assertTrue( + "Prometheus output should contains solr_metrics_core_requests for collection2", + output + .lines() + .anyMatch( + line -> + line.startsWith("solr_metrics_core_requests") + && line.contains("collection2"))); + } + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index c475f2b23d3..53e1915f7c2 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -25,7 +25,6 @@ import static org.hamcrest.core.StringContains.containsString; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; -import io.opentelemetry.api.GlobalOpenTelemetry; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Reader; @@ -113,6 +112,7 @@ import org.apache.solr.common.util.XML; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoresLocator; import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.OpenTelemetryConfigurator; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrXmlConfig; @@ -250,7 +250,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase { @BeforeClass public static void setupTestCases() { - GlobalOpenTelemetry.resetForTest(); + OpenTelemetryConfigurator.resetForTest(); resetExceptionIgnores(); testExecutor = diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 4a894521e24..b559a233a80 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -1169,6 +1169,7 @@ public class MiniSolrCloudCluster { System.setProperty("solr.cloud.overseer.enabled", Boolean.toString(overseerEnabled)); if (!disableTraceIdGeneration && OpenTelemetryConfigurator.TRACE_ID_GEN_ENABLED) { + OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, null); injectRandomRecordingFlag(); }
