This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 4aaf68b40fb SOLR-18124: Add tracing spans for UpdateLog replay (#4962)
4aaf68b40fb is described below
commit 4aaf68b40fb08b867fe865892d295fc1502afefc
Author: David Smiley <[email protected]>
AuthorDate: Wed Oct 7 14:26:25 2026 -0400
SOLR-18124: Add tracing spans for UpdateLog replay (#4962)
Co-authored-by: Kamlendra <[email protected]>
Co-authored-by: openworld-maker
<[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../SOLR-18124-updatelog-replay-tracing.yml | 9 +++++
.../src/java/org/apache/solr/update/UpdateLog.java | 45 +++++++++++++++++++++-
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml
b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml
new file mode 100644
index 00000000000..655aed1a3a4
--- /dev/null
+++ b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml
@@ -0,0 +1,9 @@
+title: Tracing (spans) for replication, recovery, and updatelog replay.
+type: changed
+authors:
+ - name: Gaurav Singh
+ nick: gsingh-code
+ - name: David Smiley
+links:
+ - name: SOLR-18124
+ url: https://issues.apache.org/jira/browse/SOLR-18124
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 2b0f202a4bd..cfe671f36df 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -22,6 +22,8 @@ import static
org.apache.solr.update.processor.DistributingUpdateProcessorFactor
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.StatusCode;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -93,6 +95,7 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.apache.solr.util.tracing.TraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2129,7 +2132,16 @@ public class UpdateLog implements PluginInfoInitialized,
SolrMetricProducer {
// setting request info will help logging
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
- try {
+ final Span span =
TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay").startSpan();
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ TraceUtils.setDbInstance(s, req.getCore().getName());
+ s.setAttribute("updatelog.replay.state", state.toString());
+ s.setAttribute("updatelog.replay.logs", translogs.size());
+ });
+ try (var scope = span.makeCurrent()) {
+ assert scope != null; // prevent javac warning about scope being unused
for (; ; ) {
TransactionLog translog = translogs.pollFirst();
if (translog == null) break;
@@ -2143,9 +2155,11 @@ public class UpdateLog implements PluginInfoInitialized,
SolrMetricProducer {
recoveryInfo.errors.incrementAndGet();
log.error("Replay failed due to exception", e);
}
+ span.recordException(e);
} catch (Exception e) {
recoveryInfo.errors.incrementAndGet();
log.error("Replay failed due to exception", e);
+ span.recordException(e);
} finally {
// change the state while updates are still blocked to prevent races
state = State.ACTIVE;
@@ -2159,6 +2173,19 @@ public class UpdateLog implements PluginInfoInitialized,
SolrMetricProducer {
log.error("ERROR: didn't get to recover from tlog {}", translog);
translog.decref();
}
+
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ s.setAttribute("updatelog.replay.adds", recoveryInfo.adds);
+ s.setAttribute("updatelog.replay.deletes", recoveryInfo.deletes);
+ s.setAttribute("updatelog.replay.deleteByQuery",
recoveryInfo.deleteByQuery);
+ s.setAttribute("updatelog.replay.errors",
recoveryInfo.errors.get());
+ });
+ if (recoveryInfo.failed || recoveryInfo.errors.get() > 0) {
+ span.setStatus(StatusCode.ERROR);
+ }
+ span.end();
}
loglog.warn("Log replay finished. recoveryInfo={}", recoveryInfo);
@@ -2169,7 +2196,18 @@ public class UpdateLog implements PluginInfoInitialized,
SolrMetricProducer {
}
public void doReplay(TransactionLog translog) {
- try {
+ final Span span =
+
TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay.log").startSpan();
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ if (translog.tlog != null) {
+ s.setAttribute("updatelog.replay.logFile",
translog.tlog.getFileName().toString());
+ }
+ s.setAttribute("updatelog.replay.logSizeBytes",
translog.getLogSize());
+ });
+ try (var scope = span.makeCurrent()) {
+ assert scope != null; // prevent javac warning about scope being unused
loglog.warn(
"Starting log replay {} active={} starting pos={}
inSortedOrder={}",
translog,
@@ -2346,10 +2384,12 @@ public class UpdateLog implements
PluginInfoInitialized, SolrMetricProducer {
} catch (ClassCastException cl) {
recoveryInfo.errors.incrementAndGet();
loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log.
Entry={}", o, cl);
+ span.recordException(cl);
// would be caused by a corrupt transaction log
} catch (Exception ex) {
recoveryInfo.errors.incrementAndGet();
loglog.warn("REPLAY_ERR: Exception replaying log", ex);
+ span.recordException(ex);
// something wrong with the request?
}
assert TestInjection.injectUpdateLogReplayRandomPause();
@@ -2393,6 +2433,7 @@ public class UpdateLog implements PluginInfoInitialized,
SolrMetricProducer {
} finally {
if (tlogReader != null) tlogReader.close();
translog.decref();
+ span.end();
}
}
