Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/1054/ Java: 64bit/jdk-10-ea+32 -XX:+UseCompressedOops -XX:+UseSerialGC
5 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest Error Message: SolrCore.getOpenCount()==2 Stack Trace: java.lang.RuntimeException: SolrCore.getOpenCount()==2 at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0) at org.apache.solr.util.TestHarness.close(TestHarness.java:379) at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:792) at org.apache.solr.cloud.AbstractZkTestCase.azt_afterClass(AbstractZkTestCase.java:147) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest Error Message: SolrCore.getOpenCount()==2 Stack Trace: java.lang.RuntimeException: SolrCore.getOpenCount()==2 at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0) at org.apache.solr.util.TestHarness.close(TestHarness.java:379) at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:792) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:288) at jdk.internal.reflect.GeneratedMethodAccessor39.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest Error Message: 28 threads leaked from SUITE scope at org.apache.solr.cloud.BasicZkTest: 1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=RUNNABLE, group=TGRP-BasicZkTest] at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89) at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 2) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 3) Thread[id=16184, name=SessionTracker, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 4) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 5) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 6) Thread[id=16156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 7) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 8) Thread[id=16164, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 9) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 10) Thread[id=16161, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-EventThread, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 11) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 12) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 13) Thread[id=16183, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:46239, state=RUNNABLE, group=TGRP-BasicZkTest] at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89) at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100) at app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 14) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 15) Thread[id=16165, name=OverseerCollectionConfigSetProcessor-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463) at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 16) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 17) Thread[id=16186, name=ProcessThread(sid:0 cport:46239):, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122) 18) Thread[id=16182, name=Thread-6525, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Thread.join(Thread.java:1353) at java.base@10-ea/java.lang.Thread.join(Thread.java:1427) at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297) at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310) at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491) 19) Thread[id=16166, name=OverseerAutoScalingTriggerThread-99241951144050691-88.99.242.108:8983_-n_0000000000, state=WAITING, group=Overseer autoscaling triggers] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 20) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 21) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 22) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 23) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 24) Thread[id=16163, name=OverseerStateUpdate-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463) at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 25) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Object.wait(Object.java:328) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719) 26) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 27) Thread[id=16155, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 28) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 28 threads leaked from SUITE scope at org.apache.solr.cloud.BasicZkTest: 1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=RUNNABLE, group=TGRP-BasicZkTest] at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89) at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 2) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 3) Thread[id=16184, name=SessionTracker, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 4) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 5) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 6) Thread[id=16156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 7) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 8) Thread[id=16164, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 9) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 10) Thread[id=16161, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-EventThread, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 11) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 12) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 13) Thread[id=16183, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:46239, state=RUNNABLE, group=TGRP-BasicZkTest] at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265) at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89) at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100) at app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 14) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 15) Thread[id=16165, name=OverseerCollectionConfigSetProcessor-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463) at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 16) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 17) Thread[id=16186, name=ProcessThread(sid:0 cport:46239):, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122) 18) Thread[id=16182, name=Thread-6525, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Thread.join(Thread.java:1353) at java.base@10-ea/java.lang.Thread.join(Thread.java:1427) at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297) at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310) at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491) 19) Thread[id=16166, name=OverseerAutoScalingTriggerThread-99241951144050691-88.99.242.108:8983_-n_0000000000, state=WAITING, group=Overseer autoscaling triggers] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 20) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 21) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 22) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 23) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 24) Thread[id=16163, name=OverseerStateUpdate-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463) at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 25) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Object.wait(Object.java:328) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719) 26) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 27) Thread[id=16155, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 28) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 2) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 3) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 4) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 5) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 6) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 7) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 8) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Object.wait(Object.java:328) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719) 14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 2) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 3) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 4) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 5) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 6) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 7) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 8) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/java.lang.Object.wait(Native Method) at java.base@10-ea/java.lang.Object.wait(Object.java:328) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719) 14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) 17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest] at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@10-ea/java.lang.Thread.run(Thread.java:844) at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0) FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalFailedMove Error Message: Error from server at http://127.0.0.1:37913/solr: create the collection time out:180s Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37913/solr: create the collection time out:180s at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480:43B512F177171A99]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:261) at org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalFailedMove(MoveReplicaHDFSTest.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Build Log: [...truncated 12511 lines...] [junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest [junit4] 2> 735000 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSTest_64BE0DB95B558480-001/init-core-data-001 [junit4] 2> 735000 WARN (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=128 numCloses=128 [junit4] 2> 735000 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 735001 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0) [junit4] 2> 735001 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSTest_64BE0DB95B558480-001/tempDir-001 [junit4] 2> 735001 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 735005 INFO (Thread-2061) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 735005 INFO (Thread-2061) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 735008 ERROR (Thread-2061) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 735105 INFO (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:43257 [junit4] 2> 735106 INFO (zkConnectionManagerCallback-7409-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected [junit4] 2> 735108 INFO (jetty-launcher-7406-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 735108 INFO (jetty-launcher-7406-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 735108 INFO (jetty-launcher-7406-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 735108 INFO (jetty-launcher-7406-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 735109 INFO (jetty-launcher-7406-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c42692{/solr,null,AVAILABLE} [junit4] 2> 735110 INFO (jetty-launcher-7406-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1107d261{/solr,null,AVAILABLE} [junit4] 2> 735110 INFO (jetty-launcher-7406-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@390723a4{/solr,null,AVAILABLE} [junit4] 2> 735110 INFO (jetty-launcher-7406-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4bc23d92{/solr,null,AVAILABLE} [junit4] 2> 735111 INFO (jetty-launcher-7406-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@649ff077{HTTP/1.1,[http/1.1]}{127.0.0.1:41083} [junit4] 2> 735111 INFO (jetty-launcher-7406-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@cc420f1{HTTP/1.1,[http/1.1]}{127.0.0.1:43661} [junit4] 2> 735111 INFO (jetty-launcher-7406-thread-1) [ ] o.e.j.s.Server Started @738095ms [junit4] 2> 735111 INFO (jetty-launcher-7406-thread-2) [ ] o.e.j.s.Server Started @738095ms [junit4] 2> 735111 INFO (jetty-launcher-7406-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41083} [junit4] 2> 7 [...truncated too long message...] ecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) [junit4] > at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) [junit4] > at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/java.lang.Object.wait(Native Method) [junit4] > at java.base@10-ea/java.lang.Object.wait(Object.java:328) [junit4] > at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719) [junit4] > 14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) [junit4] > at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > 17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest] [junit4] > at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) [junit4] > at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074) [junit4] > at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) [junit4] > at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121) [junit4] > at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [junit4] > at java.base@10-ea/java.lang.Thread.run(Thread.java:844) [junit4] > at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0) [junit4] Completed [381/764 (2!)] on J0 in 31.36s, 1 test, 4 errors <<< FAILURES! [...truncated 42069 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org