This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch branch_10_1 in repository https://gitbox.apache.org/repos/asf/solr.git
commit 8596906fb00d7d8028cd07681aa97dc6a7321312 Author: David Smiley <[email protected]> AuthorDate: Wed Oct 7 14:26:25 2026 -0400 SOLR-18124: Add tracing spans for UpdateLog replay (#4962) Co-authored-by: Kamlendra <[email protected]> Co-authored-by: openworld-maker <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]> (cherry picked from commit 4aaf68b40fb08b867fe865892d295fc1502afefc) (cherry picked from commit 394a1e28ad7a28b1422fe96d7e8354006469fb49) --- .../SOLR-18124-updatelog-replay-tracing.yml | 9 +++++ .../src/java/org/apache/solr/update/UpdateLog.java | 45 +++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml new file mode 100644 index 00000000000..655aed1a3a4 --- /dev/null +++ b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml @@ -0,0 +1,9 @@ +title: Tracing (spans) for replication, recovery, and updatelog replay. +type: changed +authors: + - name: Gaurav Singh + nick: gsingh-code + - name: David Smiley +links: + - name: SOLR-18124 + url: https://issues.apache.org/jira/browse/SOLR-18124 diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 2b0f202a4bd..cfe671f36df 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -22,6 +22,8 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; @@ -93,6 +95,7 @@ import org.apache.solr.util.RefCounted; import org.apache.solr.util.TestInjection; import org.apache.solr.util.TimeOut; import org.apache.solr.util.plugin.PluginInfoInitialized; +import org.apache.solr.util.tracing.TraceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2129,7 +2132,16 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { // setting request info will help logging SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); - try { + final Span span = TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay").startSpan(); + TraceUtils.ifNotNoop( + span, + s -> { + TraceUtils.setDbInstance(s, req.getCore().getName()); + s.setAttribute("updatelog.replay.state", state.toString()); + s.setAttribute("updatelog.replay.logs", translogs.size()); + }); + try (var scope = span.makeCurrent()) { + assert scope != null; // prevent javac warning about scope being unused for (; ; ) { TransactionLog translog = translogs.pollFirst(); if (translog == null) break; @@ -2143,9 +2155,11 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { recoveryInfo.errors.incrementAndGet(); log.error("Replay failed due to exception", e); } + span.recordException(e); } catch (Exception e) { recoveryInfo.errors.incrementAndGet(); log.error("Replay failed due to exception", e); + span.recordException(e); } finally { // change the state while updates are still blocked to prevent races state = State.ACTIVE; @@ -2159,6 +2173,19 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { log.error("ERROR: didn't get to recover from tlog {}", translog); translog.decref(); } + + TraceUtils.ifNotNoop( + span, + s -> { + s.setAttribute("updatelog.replay.adds", recoveryInfo.adds); + s.setAttribute("updatelog.replay.deletes", recoveryInfo.deletes); + s.setAttribute("updatelog.replay.deleteByQuery", recoveryInfo.deleteByQuery); + s.setAttribute("updatelog.replay.errors", recoveryInfo.errors.get()); + }); + if (recoveryInfo.failed || recoveryInfo.errors.get() > 0) { + span.setStatus(StatusCode.ERROR); + } + span.end(); } loglog.warn("Log replay finished. recoveryInfo={}", recoveryInfo); @@ -2169,7 +2196,18 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { } public void doReplay(TransactionLog translog) { - try { + final Span span = + TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay.log").startSpan(); + TraceUtils.ifNotNoop( + span, + s -> { + if (translog.tlog != null) { + s.setAttribute("updatelog.replay.logFile", translog.tlog.getFileName().toString()); + } + s.setAttribute("updatelog.replay.logSizeBytes", translog.getLogSize()); + }); + try (var scope = span.makeCurrent()) { + assert scope != null; // prevent javac warning about scope being unused loglog.warn( "Starting log replay {} active={} starting pos={} inSortedOrder={}", translog, @@ -2346,10 +2384,12 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { } catch (ClassCastException cl) { recoveryInfo.errors.incrementAndGet(); loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log. Entry={}", o, cl); + span.recordException(cl); // would be caused by a corrupt transaction log } catch (Exception ex) { recoveryInfo.errors.incrementAndGet(); loglog.warn("REPLAY_ERR: Exception replaying log", ex); + span.recordException(ex); // something wrong with the request? } assert TestInjection.injectUpdateLogReplayRandomPause(); @@ -2393,6 +2433,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { } finally { if (tlogReader != null) tlogReader.close(); translog.decref(); + span.end(); } }
