Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/822/
5 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest Error Message: 298 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 1) Thread[id=20903, name=searcherExecutor-5107-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=22223, name=coreZkRegister-4308-thread-30-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=20423, name=searcherExecutor-4604-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=22261, name=coreZkRegister-4308-thread-44-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=22325, name=coreZkRegister-4308-thread-54-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=20722, name=searcherExecutor-4962-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=20418, name=searcherExecutor-4603-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=20009, name=searcherExecutor-4452-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=20721, name=searcherExecutor-4897-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=20600, name=searcherExecutor-4836-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=22236, name=coreZkRegister-4308-thread-36-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=22199, name=coreZkRegister-4379-thread-66-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=21723, name=coreZkRegister-4308-thread-12-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=19685, name=qtp863314704-19685-selector-ServerConnectorManager@6fb78d43/0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=20419, name=searcherExecutor-4605-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=21774, name=coreZkRegister-4379-thread-44-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=20727, name=searcherExecutor-4950-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=22713, name=qtp1064931382-22713, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=22185, name=coreZkRegister-4379-thread-63-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=22201, name=coreZkRegister-4379-thread-67-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=21767, name=coreZkRegister-4379-thread-41-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=19705, name=Thread-13726, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155) 23) Thread[id=22437, name=RecoveryThread-awholynewstresscollection_collection0_0_shard1_replica1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225) 24) Thread[id=19908, name=Thread-13811, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155) 25) Thread[id=21543, name=coreZkRegister-4379-thread-25-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=20683, name=searcherExecutor-4882-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=22209, name=coreZkRegister-4379-thread-69-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=21703, name=coreZkRegister-4308-thread-3-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1024) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=22191, name=qtp863314704-22191, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=19806, name=Scheduler-202435354, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=22320, name=coreZkRegister-4308-thread-50-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=21798, name=coreZkRegister-4308-thread-25-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=22699, name=zkCallback-1125-thread-8-processing-n:127.0.0.1:41155_-EventThread, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 34) Thread[id=21763, name=coreZkRegister-4379-thread-38-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=19892, name=Scheduler-568486637, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=20507, name=searcherExecutor-4734-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=20004, name=searcherExecutor-4443-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=20967, name=searcherExecutor-5125-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=19780, name=Scheduler-955816862, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=22654, name=qtp1064931382-22654, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=21515, name=coreZkRegister-4379-thread-10-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=20707, name=searcherExecutor-4900-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=20685, name=searcherExecutor-4876-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=21888, name=coreZkRegister-4297-thread-38-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=21508, name=coreZkRegister-4379-thread-7-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=19686, name=qtp863314704-19686-selector-ServerConnectorManager@6fb78d43/1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=19808, name=Scheduler-1488862227, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=22213, name=coreZkRegister-4379-thread-71-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 49) Thread[id=20937, name=searcherExecutor-5129-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=22518, name=RecoveryThread-awholynewstresscollection_collection0_0_shard2_replica3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225) 51) Thread[id=22305, name=coreZkRegister-4308-thread-49-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=21542, name=coreZkRegister-4379-thread-24-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 53) Thread[id=21562, name=coreZkRegister-4275-thread-14, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 54) Thread[id=19635, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 55) Thread[id=20481, name=searcherExecutor-4696-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 56) Thread[id=21815, name=coreZkRegister-4379-thread-48-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 57) Thread[id=21721, name=coreZkRegister-4308-thread-10-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 58) Thread[id=20561, name=searcherExecutor-4832-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 59) Thread[id=20635, name=searcherExecutor-4842-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 60) Thread[id=20710, name=searcherExecutor-4934-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 61) Thread[id=19765, name=searcherExecutor-4303-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 62) Thread[id=21817, name=coreZkRegister-4379-thread-50-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 63) Thread[id=20942, name=searcherExecutor-5127-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 64) Thread[id=21389, name=zkCallback-1120-thread-8-processing-n:127.0.0.1:47648_-SendThread(127.0.0.1:36328), state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 65) Thread[id=22174, name=coreZkRegister-4379-thread-53-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=21923, name=coreZkRegister-4297-thread-55-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 67) Thread[id=22457, name=RecoveryThread-awholynewstresscollection_collection4_0_shard3_replica2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225) 68) Thread[id=19694, name=org.eclipse.jetty.server.session.HashSessionManager@543280feTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 69) Thread[id=20541, name=searcherExecutor-4779-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 70) Thread[id=20619, name=searcherExecutor-4848-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 71) Thread[id=20538, name=searcherExecutor-4774-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 72) Thread[id=21864, name=coreZkRegister-4297-thread-30-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 73) Thread[id=20425, name=searcherExecutor-4613-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 74) Thread[id=21692, name=coreZkRegister-4308-thread-2-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1024) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 75) Thread[id=22973, name=zkCallback-1120-thread-66, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 76) Thread[id=19885, name=qtp1498626839-19885-selector-ServerConnectorManager@20bdaaac/1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 77) Thread[id=20003, name=searcherExecutor-4442-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 78) Thread[id=20542, name=searcherExecutor-4785-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 79) Thread[id=19914, name=searcherExecutor-4385-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 80) Thread[id=20010, name=searcherExecutor-4453-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 81) Thread[id=20628, name=searcherExecutor-4860-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 82) Thread[id=22773, name=qtp863314704-22773, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 83) Thread[id=22178, name=coreZkRegister-4379-thread-57-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 84) Thread[id=19634, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 85) Thread[id=19772, name=qtp2028463532-19772-selector-ServerConnectorManager@29d21925/0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 86) Thread[id=19774, name=qtp2028463532-19774-acceptor-0@376b39eb-ServerConnector@41d2a1fb{HTTP/1.1}{127.0.0.1:41155}, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 87) Thread[id=21768, name=coreZkRegister-4379-thread-42-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 88) Thread[id=22486, name=RecoveryThread-awholynewstresscollection_collection4_0_shard5_replica1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225) 89) Thread[id=23053, name=zkCallback-1112-thread-64, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 90) Thread[id=20350, name=qtp1498626839-20350, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 91) Thread[id=20949, name=searcherExecutor-5128-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 92) Thread[id=19817, name=searcherExecutor-4323-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 93) Thread[id=20608, name=searcherExecutor-4854-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 94) Thread[id=20630, name=searcherExecutor-4857-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 95) Thread[id=20454, name=searcherExecutor-4641-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 96) Thread[id=22214, name=coreZkRegister-4379-thread-72-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 97) Thread[id=20990, name=qtp1064931382-20990, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 98) Thread[id=22179, name=coreZkRegister-4379-thread-58-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 99) Thread[id=20492, name=searcherExecutor-4714-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 100) Thread[id=21805, name=coreZkRegister-4308-thread-26-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001) at org.apache.solr.cloud.ZkController.register(ZkController.java:938) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 101) Thread[id=22237, name=coreZkRegister-4308-thread-37-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:sh [...truncated too long message...] utor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 243) Thread[id=20596, name=searcherExecutor-4838-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 244) Thread[id=21550, name=coreZkRegister-4275-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 245) Thread[id=22238, name=coreZkRegister-4308-thread-38, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 246) Thread[id=20484, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 247) Thread[id=20532, name=searcherExecutor-4765-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 248) Thread[id=22974, name=zkCallback-1120-thread-67, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 249) Thread[id=20724, name=searcherExecutor-4899-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 250) Thread[id=20939, name=searcherExecutor-5120-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 251) Thread[id=22698, name=zkCallback-1125-thread-8-processing-n:127.0.0.1:41155_-SendThread(127.0.0.1:36328), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) [junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) [junit4] > at __randomizedtesting.SeedInfo.seed([358EC306AB0C651]:0) [junit4] Completed [420/552] on J0 in 1725.93s, 1 test, 3 errors <<< FAILURES! [...truncated 412 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:782: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:719: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:59: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:233: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:516: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1432: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:987: There were test failures: 552 suites (8 ignored), 2170 tests, 2 suite-level errors, 2 errors, 1 failure, 408 ignored (17 assumptions) [seed: 358EC306AB0C651] Total time: 190 minutes 42 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts No prior successful build to compare, so performing full copy of artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org