[
https://issues.apache.org/jira/browse/SOLR-17463?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jerry Chung updated SOLR-17463:
-------------------------------
Labels: deadlock metrics monitoring (was: )
> Solr hanging at SolrCore.getSearcher
> ------------------------------------
>
> Key: SOLR-17463
> URL: https://issues.apache.org/jira/browse/SOLR-17463
> Project: Solr
> Issue Type: Bug
> Security Level: Public(Default Security Level. Issues are Public)
> Components: search
> Affects Versions: 9.6.1
> Reporter: Jerry Chung
> Priority: Blocker
> Labels: deadlock, metrics, monitoring
>
> Related Issue: https://issues.apache.org/jira/browse/SOLR-17060
> It was not fully fixed with the above ticket and we are still observing the
> similar issue
> java.lang.InterruptedException: null
> at java.base/java.lang.Object.wait(Native Method) ~[?:?]
> at java.base/java.lang.Object.wait(Object.java:338) ~[?:?]
> at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2523)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2266)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2100)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.apache.solr.core.SolrCore.withSearcher(SolrCore.java:2119)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.apache.solr.core.SolrCore.getSegmentCount(SolrCore.java:540)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.core.SolrCore.lambda$initializeMetrics$12(SolrCore.java:1347)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.util.stats.MetricUtils.convertGauge(MetricUtils.java:657)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.util.stats.MetricUtils.convertMetric(MetricUtils.java:355)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.util.stats.MetricUtils.lambda$toMaps$6(MetricUtils.java:250)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> ~[?:?]
> at
> java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
> ~[?:?]
> at
> java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
> ~[?:?]
> at
> java.base/java.util.TreeMap$KeySpliterator.forEachRemaining(TreeMap.java:3064)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
> ~[?:?]
> at
> java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> ~[?:?]
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> ~[?:?]
> at
> java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
> ~[?:?]
> at org.apache.solr.util.stats.MetricUtils.toMaps(MetricUtils.java:247)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.handler.admin.MetricsHandler.handleExprRequest(MetricsHandler.java:227)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.handler.admin.MetricsHandler.handleRequest(MetricsHandler.java:133)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.handler.admin.MetricsHandler.handleRequestBody(MetricsHandler.java:113)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:226)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:967)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:915)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:576)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:262)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:219)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:249)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:215)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:213)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:195)
> ~[solr-core-9.6.1.jar:9.6.1 d7f7166567f52f1b31e3315b0188e11f2c4c9b60 -
> houston - 2024-05-23 13:50:22]
> at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:210)
> ~[jetty-servlet-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
> ~[jetty-servlet-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
> ~[jetty-servlet-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:598)
> ~[jetty-security-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1580)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
> ~[jetty-servlet-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1553)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:149)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:228)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:141)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
> ~[jetty-rewrite-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:822)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.server.Server.handle(Server.java:563)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.server.HttpChannel$RequestDispatchable.dispatch(HttpChannel.java:1598)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:753)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:501)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:461)
> ~[jetty-server-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:193)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.http2.HTTP2Connection.produce(HTTP2Connection.java:208)
> ~[http2-common-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.http2.HTTP2Connection.onFillable(HTTP2Connection.java:155)
> ~[http2-common-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.http2.HTTP2Connection$FillableCallback.succeeded(HTTP2Connection.java:450)
> ~[http2-common-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:558)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:379)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:146)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
> ~[jetty-io-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:199)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
> ~[jetty-util-10.0.20.jar:10.0.20]
> at java.base/java.lang.Thread.run(Thread.java:840) [?:?]
>
> Thousands of threads are waiting on this lock and Solr is almost inaccessible.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org
For additional commands, e-mail: issues-h...@solr.apache.org
