This is an automated email from the ASF dual-hosted git repository.
mlbiscoc pushed a commit to branch feature/SOLR-17458
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/feature/SOLR-17458 by this
push:
new a933b0496f2 Migrate FieldCacheMetrics to OTEL (#3619)
a933b0496f2 is described below
commit a933b0496f28048162c97f66bc23e42054a3b50d
Author: Matthew Biscocho <[email protected]>
AuthorDate: Fri Sep 19 16:06:06 2025 -0400
Migrate FieldCacheMetrics to OTEL (#3619)
Co-authored-by: Matthew Biscocho <[email protected]>
---
.../java/org/apache/solr/core/CoreContainer.java | 6 +-
.../src/java/org/apache/solr/core/SolrCore.java | 34 +++----
.../org/apache/solr/search/SolrFieldCacheBean.java | 37 ++++---
.../apache/solr/uninverting/UninvertingReader.java | 8 +-
.../apache/solr/search/TestSolrFieldCacheBean.java | 108 ++++++++++++---------
5 files changed, 109 insertions(+), 84 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 737a8628be4..8a322cc8cbd 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1109,8 +1109,10 @@ public class CoreContainer {
"version");
SolrFieldCacheBean fieldCacheBean = new SolrFieldCacheBean();
- // NOCOMMIT SOLR-17458: Otel migration
- fieldCacheBean.initializeMetrics(solrMetricsContext, Attributes.empty(),
"");
+ fieldCacheBean.initializeMetrics(
+ solrMetricsContext,
+ Attributes.of(CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString()),
+ "");
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java
b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 97b0f426678..c62563a2209 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -259,6 +259,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
private volatile boolean newSearcherReady = false;
+ private final Attributes coreAttributes;
private AttributedLongCounter newSearcherCounter;
private AttributedLongCounter newSearcherMaxReachedCounter;
private AttributedLongCounter newSearcherOtherErrorsCounter;
@@ -1088,6 +1089,17 @@ public class SolrCore implements SolrInfoBean, Closeable
{
this.solrMetricsContext = coreMetricManager.getSolrMetricsContext();
this.coreMetricManager.loadReporters();
+ if (coreContainer.isZooKeeperAware()) {
+ this.coreAttributes =
+ Attributes.builder()
+ .put(COLLECTION_ATTR, coreDescriptor.getCollectionName())
+ .put(CORE_ATTR, coreDescriptor.getName())
+ .put(SHARD_ATTR,
coreDescriptor.getCloudDescriptor().getShardId())
+ .build();
+ } else {
+ this.coreAttributes = Attributes.builder().put(CORE_ATTR,
coreDescriptor.getName()).build();
+ }
+
if (updateHandler == null) {
directoryFactory = initDirectoryFactory();
recoveryStrategyBuilder = initRecoveryStrategyBuilder();
@@ -1109,21 +1121,7 @@ public class SolrCore implements SolrInfoBean, Closeable
{
setLatestSchema(schema);
// initialize core metrics
- if (coreContainer.isZooKeeperAware()) {
- initializeMetrics(
- solrMetricsContext,
- Attributes.builder()
- .put(COLLECTION_ATTR, coreDescriptor.getCollectionName())
- .put(CORE_ATTR, coreDescriptor.getName())
- .put(SHARD_ATTR,
coreDescriptor.getCloudDescriptor().getShardId())
- .build(),
- "");
- } else {
- initializeMetrics(
- solrMetricsContext,
- Attributes.builder().put(CORE_ATTR,
coreDescriptor.getName()).build(),
- "");
- }
+ initializeMetrics(solrMetricsContext, coreAttributes, "");
// init pluggable circuit breakers, after metrics because some circuit
breakers use metrics
initPlugins(null, CircuitBreaker.class);
@@ -1131,8 +1129,10 @@ public class SolrCore implements SolrInfoBean, Closeable
{
SolrFieldCacheBean solrFieldCacheBean = new SolrFieldCacheBean();
// this is registered at the CONTAINER level because it's not
core-specific - for now we
// also register it here for back-compat
- // NOCOMMIT SOLR-17458: Add Otel
- solrFieldCacheBean.initializeMetrics(solrMetricsContext,
Attributes.empty(), "core");
+ solrFieldCacheBean.initializeMetrics(
+ solrMetricsContext,
+ coreAttributes.toBuilder().put(CATEGORY_ATTR,
Category.CACHE.toString()).build(),
+ "");
infoRegistry.put("fieldCache", solrFieldCacheBean);
this.maxWarmingSearchers = solrConfig.maxWarmingSearchers;
diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
index 58c1bdb55c1..a0dff1245d7 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
@@ -17,9 +17,10 @@
package org.apache.solr.search;
import io.opentelemetry.api.common.Attributes;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.metrics.otel.OtelUnit;
import org.apache.solr.uninverting.UninvertingReader;
/** A SolrInfoBean that provides introspection of the Solr FieldCache */
@@ -28,6 +29,7 @@ public class SolrFieldCacheBean implements SolrInfoBean {
private boolean disableEntryList =
Boolean.getBoolean("disableSolrFieldCacheMBeanEntryList");
private boolean disableJmxEntryList =
Boolean.getBoolean("disableSolrFieldCacheMBeanEntryListJmx");
+ private AutoCloseable toClose;
private SolrMetricsContext solrMetricsContext;
@@ -51,28 +53,35 @@ public class SolrFieldCacheBean implements SolrInfoBean {
return solrMetricsContext;
}
- // TODO SOLR-17458: Migrate to Otel
@Override
public void initializeMetrics(
SolrMetricsContext parentContext, Attributes attributes, String scope) {
this.solrMetricsContext = parentContext;
- MetricsMap metricsMap =
- new MetricsMap(
- map -> {
+ var solrCacheStats =
+ solrMetricsContext.longMeasurement(
+ "solr_field_cache_entries", "Number of field cache entries");
+ var solrCacheSize =
+ solrMetricsContext.longMeasurement(
+ "solr_field_cache_size", "Size of field cache in bytes",
OtelUnit.BYTES);
+ this.toClose =
+ solrMetricsContext.batchCallback(
+ () -> {
if (!disableEntryList && !disableJmxEntryList) {
UninvertingReader.FieldCacheStats fieldCacheStats =
UninvertingReader.getUninvertedStats();
String[] entries = fieldCacheStats.info;
- map.put("entries_count", entries.length);
- map.put("total_size", fieldCacheStats.totalSize);
- for (int i = 0; i < entries.length; i++) {
- final String entry = entries[i];
- map.put("entry#" + i, entry);
- }
+ solrCacheStats.record(entries.length, attributes);
+ solrCacheSize.record(fieldCacheStats.totalSize, attributes);
} else {
- map.put("entries_count",
UninvertingReader.getUninvertedStatsSize());
+
solrCacheStats.record(UninvertingReader.getUninvertedStatsSize(), attributes);
}
- });
- solrMetricsContext.gauge(metricsMap, true, "fieldCache",
Category.CACHE.toString(), scope);
+ },
+ solrCacheStats,
+ solrCacheSize);
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(toClose);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index c0f4f209921..5f85aac034e 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -38,7 +38,6 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.uninverting.FieldCache.CacheEntry;
/**
@@ -467,8 +466,7 @@ public class UninvertingReader extends FilterLeafReader {
info[i] = entries[i].toString();
totalBytesUsed += entries[i].getValue().ramBytesUsed();
}
- String totalSize = RamUsageEstimator.humanReadableUnits(totalBytesUsed);
- return new FieldCacheStats(totalSize, info);
+ return new FieldCacheStats(totalBytesUsed, info);
}
public static int getUninvertedStatsSize() {
@@ -481,10 +479,10 @@ public class UninvertingReader extends FilterLeafReader {
* @lucene.internal
*/
public static class FieldCacheStats {
- public String totalSize;
+ public Long totalSize;
public String[] info;
- public FieldCacheStats(String totalSize, String[] info) {
+ public FieldCacheStats(Long totalSize, String[] info) {
this.totalSize = totalSize;
this.info = info;
}
diff --git
a/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheBean.java
b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheBean.java
index c7d5097a9e6..fea83d51038 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheBean.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheBean.java
@@ -16,12 +16,14 @@
*/
package org.apache.solr.search;
+import static org.apache.solr.metrics.SolrMetricProducer.CATEGORY_ATTR;
+
import io.opentelemetry.api.common.Attributes;
-import java.util.Map;
-import java.util.Random;
+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
+import java.util.Optional;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricsContext;
import org.junit.BeforeClass;
@@ -29,6 +31,12 @@ import org.junit.Test;
public class TestSolrFieldCacheBean extends SolrTestCaseJ4 {
+ private static final String ENTRIES_METRIC_NAME = "solr_field_cache_entries";
+ private static final String SIZE_BYTES_METRIC_NAME =
"solr_field_cache_size_bytes";
+ private static final String DISABLE_ENTRY_LIST_PROPERTY =
"disableSolrFieldCacheMBeanEntryList";
+ private static final String DISABLE_ENTRY_LIST_JMX_PROPERTY =
+ "disableSolrFieldCacheMBeanEntryListJmx";
+
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema-minimal.xml");
@@ -36,69 +44,77 @@ public class TestSolrFieldCacheBean extends SolrTestCaseJ4 {
@Test
public void testEntryList() {
- // ensure entries to FieldCache
+ // Ensure entries to FieldCache
assertU(adoc("id", "id0"));
assertU(commit());
assertQ(req("q", "*:*", "sort", "id asc"), "//*[@numFound='1']");
// Test with entry list enabled
- assertEntryListIncluded(false);
+ assertEntryList(true);
- // Test again with entry list disabled
- System.setProperty("disableSolrFieldCacheMBeanEntryList", "true");
+ // Test with entry list disabled
+ System.setProperty(DISABLE_ENTRY_LIST_PROPERTY, "true");
try {
- assertEntryListNotIncluded(false);
+ assertEntryList(false);
} finally {
- System.clearProperty("disableSolrFieldCacheMBeanEntryList");
+ System.clearProperty(DISABLE_ENTRY_LIST_PROPERTY);
}
- // Test with entry list enabled for jmx
- assertEntryListIncluded(true);
+ // Test with entry list enabled again
+ assertEntryList(true);
- // Test with entry list disabled for jmx
- System.setProperty("disableSolrFieldCacheMBeanEntryListJmx", "true");
+ // Test with entry list disabled for JMX
+ System.setProperty(DISABLE_ENTRY_LIST_JMX_PROPERTY, "true");
try {
- assertEntryListNotIncluded(true);
+ assertEntryList(false);
} finally {
- System.clearProperty("disableSolrFieldCacheMBeanEntryListJmx");
+ System.clearProperty(DISABLE_ENTRY_LIST_JMX_PROPERTY);
}
}
- private void assertEntryListIncluded(boolean checkJmx) {
- SolrFieldCacheBean mbean = new SolrFieldCacheBean();
- Random r = random();
- String registryName = TestUtil.randomSimpleString(r, 1, 10);
- SolrMetricManager metricManager = h.getCoreContainer().getMetricManager();
- SolrMetricsContext solrMetricsContext =
- new SolrMetricsContext(metricManager, registryName, "foo");
- mbean.initializeMetrics(solrMetricsContext, Attributes.empty(), null);
- MetricsMap metricsMap =
- (MetricsMap)
- ((SolrMetricManager.GaugeWrapper)
-
metricManager.registry(registryName).getMetrics().get("CACHE.fieldCache"))
- .getGauge();
- Map<String, Object> metrics = checkJmx ? metricsMap.getValue(true) :
metricsMap.getValue();
- assertTrue(((Number) metrics.get("entries_count")).longValue() > 0);
- assertNotNull(metrics.get("total_size"));
- assertNotNull(metrics.get("entry#0"));
+ private void assertEntryList(boolean bytesMetricIncluded) {
+ FieldCacheMetrics metrics = getFieldCacheMetrics();
+ assertTrue(
+ "Field cache entries count should be greater than 0",
+ metrics.entries().get().getDataPoints().getFirst().getValue() > 0);
+
+ if (bytesMetricIncluded) {
+ assertTrue("Size bytes metric should be present",
metrics.sizeBytes().isPresent());
+ } else {
+ assertTrue("Size bytes metric should not be present",
metrics.sizeBytes().isEmpty());
+ }
}
- private void assertEntryListNotIncluded(boolean checkJmx) {
- SolrFieldCacheBean mbean = new SolrFieldCacheBean();
- Random r = random();
- String registryName = TestUtil.randomSimpleString(r, 1, 10);
+ private FieldCacheMetrics getFieldCacheMetrics() {
+ String registryName = TestUtil.randomSimpleString(random(), 1, 10);
SolrMetricManager metricManager = h.getCoreContainer().getMetricManager();
SolrMetricsContext solrMetricsContext =
new SolrMetricsContext(metricManager, registryName, "foo");
- mbean.initializeMetrics(solrMetricsContext, Attributes.empty(), null);
- MetricsMap metricsMap =
- (MetricsMap)
- ((SolrMetricManager.GaugeWrapper)
-
metricManager.registry(registryName).getMetrics().get("CACHE.fieldCache"))
- .getGauge();
- Map<String, Object> metrics = checkJmx ? metricsMap.getValue(true) :
metricsMap.getValue();
- assertTrue(((Number) metrics.get("entries_count")).longValue() > 0);
- assertNull(metrics.get("total_size"));
- assertNull(metrics.get("entry#0"));
+
+ try (SolrFieldCacheBean mbean = new SolrFieldCacheBean()) {
+ mbean.initializeMetrics(
+ solrMetricsContext,
+ Attributes.of(CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString()),
+ null);
+
+ var metrics =
metricManager.getPrometheusMetricReader(registryName).collect();
+
+ var entryCount =
+ metrics.stream()
+ .filter(ms ->
ENTRIES_METRIC_NAME.equals(ms.getMetadata().getPrometheusName()))
+ .map(GaugeSnapshot.class::cast)
+ .findFirst();
+
+ var sizeBytes =
+ metrics.stream()
+ .filter(ms ->
SIZE_BYTES_METRIC_NAME.equals(ms.getMetadata().getPrometheusName()))
+ .map(GaugeSnapshot.class::cast)
+ .findFirst();
+
+ return new FieldCacheMetrics(entryCount, sizeBytes);
+ }
}
+
+ private record FieldCacheMetrics(
+ Optional<GaugeSnapshot> entries, Optional<GaugeSnapshot> sizeBytes) {}
}
