Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2652/
4 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates Error Message: Error from server at http://127.0.0.1:34269/solr: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Not Found</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/admin/collections. Reason: <pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down. at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:724) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) </pre> <hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34269/solr: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Not Found</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/admin/collections. Reason: <pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down. at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:724) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) </pre> <hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> at __randomizedtesting.SeedInfo.seed([4E3987ED6D2F7DC0]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211) at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.createMiniSolrCloudCluster(TestStressCloudBlindAtomicUpdates.java:133) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates Error Message: ObjectTracker found 9 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, SolrZkClient, SolrZkClient, InternalHttpClient, ZkController, Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:546) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:328) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:266) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:253) at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:280) at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1581) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:674) at org.apache.solr.cloud.Overseer.getSolrCloudManager(Overseer.java:570) at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.<init>(OverseerCollectionMessageHandler.java:205) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getOverseerMessageHandlerSelector(OverseerCollectionConfigSetProcessor.java:87) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:70) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:41) at org.apache.solr.cloud.Overseer.start(Overseer.java:540) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:546) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:328) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:266) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:253) at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:280) at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1581) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:674) at org.apache.solr.core.CoreContainer.createMetricsHistoryHandler(CoreContainer.java:757) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:578) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:186) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:327) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:556) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:186) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:121) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111) at org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:295) at org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:155) at org.apache.solr.client.solrj.impl.CloudSolrClient.getZkStateReader(CloudSolrClient.java:342) at org.apache.solr.client.solrj.impl.SolrClientCloudManager.<init>(SolrClientCloudManager.java:69) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:675) at org.apache.solr.cloud.Overseer.getSolrCloudManager(Overseer.java:570) at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.<init>(OverseerCollectionMessageHandler.java:205) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getOverseerMessageHandlerSelector(OverseerCollectionConfigSetProcessor.java:87) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:70) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:41) at org.apache.solr.cloud.Overseer.start(Overseer.java:540) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:215) at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:540) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:454) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:556) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:553) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: java.lang.AssertionError: ObjectTracker found 9 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, SolrZkClient, SolrZkClient, InternalHttpClient, ZkController, Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:546) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:328) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:266) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:253) at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:280) at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1581) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:674) at org.apache.solr.cloud.Overseer.getSolrCloudManager(Overseer.java:570) at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.<init>(OverseerCollectionMessageHandler.java:205) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getOverseerMessageHandlerSelector(OverseerCollectionConfigSetProcessor.java:87) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:70) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:41) at org.apache.solr.cloud.Overseer.start(Overseer.java:540) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:546) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:328) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:266) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:253) at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:280) at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1581) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:674) at org.apache.solr.core.CoreContainer.createMetricsHistoryHandler(CoreContainer.java:757) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:578) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:186) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:327) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:556) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:186) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:121) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111) at org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:295) at org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:155) at org.apache.solr.client.solrj.impl.CloudSolrClient.getZkStateReader(CloudSolrClient.java:342) at org.apache.solr.client.solrj.impl.SolrClientCloudManager.<init>(SolrClientCloudManager.java:69) at org.apache.solr.cloud.ZkController.getSolrCloudManager(ZkController.java:675) at org.apache.solr.cloud.Overseer.getSolrCloudManager(Overseer.java:570) at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.<init>(OverseerCollectionMessageHandler.java:205) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getOverseerMessageHandlerSelector(OverseerCollectionConfigSetProcessor.java:87) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:70) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:41) at org.apache.solr.cloud.Overseer.start(Overseer.java:540) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:319) at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:215) at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:540) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:454) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:113) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:556) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:741) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1477) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1542) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1186) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1023) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:473) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:308) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:396) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:369) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:392) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:248) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:553) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([4E3987ED6D2F7DC0]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:304) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates Error Message: 19 threads leaked from SUITE scope at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates: 1) Thread[id=1245, name=ScheduledTrigger-476-thread-4, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1183, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1237, name=zkCallback-349-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=1235, name=OverseerStateUpdate-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer state updater.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1213) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:341) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$33/712935252.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:341) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:419) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1236, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1157, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1240, name=OverseerCollectionConfigSetProcessor-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer collection creation process.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1213) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:341) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$33/712935252.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:341) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:393) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1243, name=ScheduledTrigger-476-thread-2, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1244, name=ScheduledTrigger-476-thread-3, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1241, name=OverseerAutoScalingTriggerThread-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=1238, name=zkCallback-349-thread-2-EventThread, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 12) Thread[id=1156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=1158, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=1162, name=jetty-launcher-317-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 15) Thread[id=1239, name=zkConnectionManagerCallback-395-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=1163, name=jetty-launcher-317-thread-2-EventThread, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 17) Thread[id=1164, name=zkConnectionManagerCallback-350-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=1242, name=ScheduledTrigger-476-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=1190, name=zkConnectionManagerCallback-371-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 19 threads leaked from SUITE scope at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates: 1) Thread[id=1245, name=ScheduledTrigger-476-thread-4, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1183, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1237, name=zkCallback-349-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=1235, name=OverseerStateUpdate-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer state updater.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1213) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:341) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$33/712935252.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:341) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:419) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1236, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1157, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1240, name=OverseerCollectionConfigSetProcessor-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer collection creation process.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1213) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:341) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$33/712935252.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:341) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:393) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1243, name=ScheduledTrigger-476-thread-2, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1244, name=ScheduledTrigger-476-thread-3, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1241, name=OverseerAutoScalingTriggerThread-73081450705059850-127.0.0.1:34269_solr-n_0000000002, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=1238, name=zkCallback-349-thread-2-EventThread, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 12) Thread[id=1156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=1158, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=1162, name=jetty-launcher-317-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 15) Thread[id=1239, name=zkConnectionManagerCallback-395-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=1163, name=jetty-launcher-317-thread-2-EventThread, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 17) Thread[id=1164, name=zkConnectionManagerCallback-350-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=1242, name=ScheduledTrigger-476-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=1190, name=zkConnectionManagerCallback-371-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([4E3987ED6D2F7DC0]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=1245, name=ScheduledTrigger-476-thread-4, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1237, name=zkCallback-349-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 3) Thread[id=1162, name=jetty-launcher-317-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=1239, name=zkConnectionManagerCallback-395-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1164, name=zkConnectionManagerCallback-350-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1243, name=ScheduledTrigger-476-thread-2, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1244, name=ScheduledTrigger-476-thread-3, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1242, name=ScheduledTrigger-476-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1190, name=zkConnectionManagerCallback-371-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=1245, name=ScheduledTrigger-476-thread-4, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1237, name=zkCallback-349-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 3) Thread[id=1162, name=jetty-launcher-317-thread-2-SendThread(127.0.0.1:43243), state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=1239, name=zkConnectionManagerCallback-395-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1164, name=zkConnectionManagerCallback-350-thread-1, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1243, name=ScheduledTrigger-476-thread-2, state=WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1244, name=ScheduledTrigger-476-thread-3, state=TIMED_WAITING, group=TGRP-TestStressCloudBlindAtomicUpdates] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolE [...truncated too long message...] -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: ivy-availability-check: [loadresource] Do not set property disallowed.ivy.jars.list as its length is 0. -ivy-fail-disallowed-ivy-version: ivy-fail: ivy-fail: ivy-configure: [ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml resolve: jar-checksums: [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null358191313 [copy] Copying 240 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null358191313 [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null358191313 check-working-copy: [ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working [ivy:cachepath] confs: [default] [ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public [ivy:cachepath] found com.jcraft#jsch;0.1.53 in public [ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public [ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public [ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public [ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public [ivy:cachepath] found commons-codec#commons-codec;1.6 in public [ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public [ivy:cachepath] :: resolution report :: resolve 104ms :: artifacts dl 1ms --------------------------------------------------------------------- | | modules || artifacts | | conf | number| search|dwnlded|evicted|| number|dwnlded| --------------------------------------------------------------------- | default | 8 | 0 | 0 | 0 || 8 | 0 | --------------------------------------------------------------------- [wc-checker] Initializing working copy... [wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder". [wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation [wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details. [wc-checker] Checking working copy status... -jenkins-base: BUILD SUCCESSFUL Total time: 158 minutes 58 seconds Archiving artifacts java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath$34.hasMatch(FilePath.java:2678) at hudson.FilePath$34.invoke(FilePath.java:2557) at hudson.FilePath$34.invoke(FilePath.java:2547) at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918) Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2 at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741) at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357) at hudson.remoting.Channel.call(Channel.java:955) at hudson.FilePath.act(FilePath.java:1036) at hudson.FilePath.act(FilePath.java:1025) at hudson.FilePath.validateAntFileMask(FilePath.java:2547) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690) at hudson.model.Build$BuildExecution.post2(Build.java:186) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635) at hudson.model.Run.execute(Run.java:1819) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) Caused: hudson.FilePath$TunneledInterruptedException at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920) at hudson.remoting.UserRequest.perform(UserRequest.java:212) at hudson.remoting.UserRequest.perform(UserRequest.java:54) at hudson.remoting.Request$2.run(Request.java:369) at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000 at hudson.FilePath.act(FilePath.java:1038) at hudson.FilePath.act(FilePath.java:1025) at hudson.FilePath.validateAntFileMask(FilePath.java:2547) at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243) at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81) at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20) at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744) at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690) at hudson.model.Build$BuildExecution.post2(Build.java:186) at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635) at hudson.model.Run.execute(Run.java:1819) at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43) at hudson.model.ResourceController.execute(ResourceController.java:97) at hudson.model.Executor.run(Executor.java:429) No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error? Recording test results Build step 'Publish JUnit test result report' changed build result to UNSTABLE Email was triggered for: Unstable (Test Failures) Sending email for trigger: Unstable (Test Failures)
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
