Hi Folks, Solr 8.4.1 - we've started doing an expungeDeletes daily at 5am for various reasons but I've also started having these log messages appear in our master from some of the followers.
There's no disk io/cpu contention, everything's running fast - so not sure: 1. Why we're getting this 2. If we need to be worried Anyone got any ideas? Karl 05:10:14.457 [updateExecutor-5-thread-3217-processing-n:solr-0.search-solr.prod.k8.atcloud.io:80_solr x:at-uk-002_shard1_replica_n6 c:at-uk-002 s:shard1 r:core_node9] ERROR org.apache.solr.update.SolrCmdDistributor - org.apache.solr.clien t.solrj.SolrServerException: Timeout occured while waiting response from server at: http://solr-5.search-solr.prod.k8.atcloud.io/solr/at-uk-002_shard1_replica_n1/update?update.distrib=FROMLEADER&distrib.from=http%3A%2F%2Fsolr-0.search-sol r.prod.k8.atcloud.io%3A80%2Fsolr%2Fat-uk-002_shard1_replica_n6%2F at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:407) at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:753) at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:369) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) at org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:344) at org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:333) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:181) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: java.util.concurrent.TimeoutException at org.eclipse.jetty.client.util.InputStreamResponseListener.get(InputStreamResponseListener.java:216) at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:398) ... 13 more 05:10:14.457 [updateExecutor-5-thread-3216-processing-n:solr-0.search-solr.prod.k8.atcloud.io:80_solr x:at-uk-002_shard1_replica_n6 c:at-uk-002 s:shard1 r:core_node9] ERROR org.apache.solr.update.SolrCmdDistributor - org.apache.solr.clien t.solrj.SolrServerException: Timeout occured while waiting response from server at: http://solr-2.search-solr.prod.k8.atcloud.io/solr/at-uk-002_shard1_replica_n10/update?update.distrib=FROMLEADER&distrib.from=http%3A%2F%2Fsolr-0.search-so lr.prod.k8.atcloud.io%3A80%2Fsolr%2Fat-uk-002_shard1_replica_n6%2F at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:407) at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:753) at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:369) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) at org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:344) at org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:333) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:181) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) Caused by: java.util.concurrent.TimeoutException at org.eclipse.jetty.client.util.InputStreamResponseListener.get(InputStreamResponseListener.java:216) at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:398) ... 13 more 05:10:14.457 [updateExecutor-5-thread-3208-processing-n:solr-0.search-solr.prod.k8.atcloud.io:80_solr x:at-uk-002_shard1_replica_n6 c:at-uk-002 s:shard1 r:core_node9] ERROR org.apache.solr.update.SolrCmdDistributor - org.apache.solr.clien t.solrj.SolrServerException: Timeout occured while waiting response from server at: http://solr-1.search-solr.prod.k8.atcloud.io/solr/at-uk-002_shard1_replica_n8/update?update.distrib=FROMLEADER&distrib.from=http%3A%2F%2Fsolr-0.search-sol r.prod.k8.atcloud.io%3A80%2Fsolr%2Fat-uk-002_shard1_replica_n6%2F at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:407) at org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:753) at org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:369) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) at org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:344) at org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:333) This e-mail is sent on behalf of Auto Trader Group Plc, Registered Office: 1 Tony Wilson Place, Manchester, Lancashire, M15 4FN (Registered in England No. 9439967). This email and any files transmitted with it are confidential and may be legally privileged, and intended solely for the use of the individual or entity to whom they are addressed. If you have received this email in error please notify the sender. This email message has been swept for the presence of computer viruses.
