Hi team, We are weirdly seeing this peculiar exception when going beyond a limit at the time of load testing for a POC. We are trying to utilise *Gatling* HTTP as a load tester documented under https://gatling.io/docs/gatling/reference/current/http/protocol/.
*2021-11-02 06:25:40.831 ERROR (qtp1070619454-49084) [c:libraryP0001 s:shard2 r:core_node7 x:libraryP0001_shard2_replica_n4] o.a.s.s.HttpSolrCall org.apache.solr.common.SolrException: org.apache.solr.client.solrj.SolrServerException: Max requests queued per destination 3000 exceeded for HttpDestination[http://solr-node1-eu.lan:8983]@72642896,queue=3000,pool=MultiplexConnectionPool@765ba202[c=0/4/4,a=0,i=0]at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:477)at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:216)at org.apache.solr.core.SolrCore.execute(SolrCore.java:2646)at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:794)at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:567)at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:357)at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201)at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:602)at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612)at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582)at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191)at org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)at org.eclipse.jetty.server.Server.handle(Server.java:516)at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556)at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375)at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)at java.base/java.lang.Thread.run(Thread.java:834)Caused by: org.apache.solr.client.solrj.SolrServerException: Max requests queued per destination 3000 exceeded for HttpDestination[http://solr-node1-eu.lan:8983]@72642896,queue=3000,pool=MultiplexConnectionPool@765ba202[c=0/4/4,a=0,i=0]at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:430)at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:761)at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)at org.apache.solr.handler.component.HttpShardHandlerFactory.makeLoadBalancedRequest(HttpShardHandlerFactory.java:371)at org.apache.solr.handler.component.ShardRequestor.call(ShardRequestor.java:132)at org.apache.solr.handler.component.ShardRequestor.call(ShardRequestor.java:41)at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)at datadog.trace.bootstrap.instrumentation.java.concurrent.Wrapper.run(Wrapper.java:46)at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)at datadog.trace.bootstrap.instrumentation.java.concurrent.Wrapper.run(Wrapper.java:46)at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)... 1 moreCaused by: java.util.concurrent.RejectedExecutionException: Max requests queued per destination 3000 exceeded for HttpDestination[http://solr-node1-eu.lan:8983]@72642896,queue=3000,pool=MultiplexConnectionPool@765ba202[c=0/4/4,a=0,i=0]at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:295)at org.eclipse.jetty.client.HttpDestination.send(HttpDestination.java:265)at org.eclipse.jetty.client.HttpClient.send(HttpClient.java:594)at org.eclipse.jetty.client.HttpRequest.send(HttpRequest.java:772)at org.eclipse.jetty.client.HttpRequest.send(HttpRequest.java:764)at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:400)... 16 more* Amrit Sarkar Engineer | Search and Kubernetes https://seamadic.com/ Twitter https://twitter.com/sarkaramrit2 LinkedIn: https://www.linkedin.com/in/sarkaramrit2 Medium: https://medium.com/@sarkaramrit2
