Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/832/
6 tests failed. FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test Error Message: GC overhead limit exceeded Stack Trace: java.lang.OutOfMemoryError: GC overhead limit exceeded FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest Error Message: 528 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 1) Thread[id=15054, name=Thread-9920, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 2) Thread[id=15303, name=Thread-10153, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 3) Thread[id=15391, name=Thread-10234, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 4) Thread[id=13065, name=searcherExecutor-3543-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=15281, name=Thread-10134, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 6) Thread[id=15344, name=Thread-10189, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 7) Thread[id=15354, name=Thread-10198, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 8) Thread[id=12935, name=searcherExecutor-3370-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=13308, name=searcherExecutor-3783-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=13934, name=zkCallback-941-thread-18-processing-n:127.0.0.1:33879_-EventThread, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 11) Thread[id=15326, name=Thread-10172, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 12) Thread[id=12123, name=qtp1631332304-12123, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=13250, name=searcherExecutor-3733-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=14260, name=qtp1827900685-14260, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=14158, name=qtp1827900685-14158, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=13044, name=searcherExecutor-3510-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=13028, name=searcherExecutor-3508-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=14986, name=Thread-9853, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 19) Thread[id=15105, name=Thread-9970, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 20) Thread[id=15192, name=Thread-10055, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 21) Thread[id=14991, name=Thread-9858, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 22) Thread[id=12054, name=Scheduler-346774206, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=11996, name=Scheduler-1448316559, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=14843, name=Thread-9713, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 25) Thread[id=15271, name=Thread-10124, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 26) Thread[id=15250, name=Thread-10104, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 27) Thread[id=12326, name=searcherExecutor-3115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=13334, name=searcherExecutor-3794-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=14882, name=Thread-9750, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 30) Thread[id=15388, name=Thread-10231, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 31) Thread[id=13613, name=searcherExecutor-3868-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=12151, name=searcherExecutor-2946-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=15403, name=Thread-10246, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 34) Thread[id=15233, name=Thread-10089, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 35) Thread[id=15257, name=Thread-10111, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 36) Thread[id=15328, name=Thread-10174, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 37) Thread[id=15203, name=Thread-10064, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 38) Thread[id=14807, name=Thread-9677, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 39) Thread[id=15027, name=Thread-9893, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 40) Thread[id=15323, name=Thread-10170, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 41) Thread[id=15280, name=Thread-10133, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 42) Thread[id=13169, name=searcherExecutor-3637-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=15210, name=Thread-10071, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 44) Thread[id=11990, name=qtp1631332304-11990-acceptor-0@2ecfc8e4-ServerConnector@206cabeb{HTTP/1.1}{127.0.0.1:33879}, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=15350, name=Thread-10194, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 46) Thread[id=15110, name=Thread-9975, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 47) Thread[id=13332, name=searcherExecutor-3785-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=15353, name=Thread-10197, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 49) Thread[id=11959, name=qtp1827900685-11959-acceptor-0@31d15da2-ServerConnector@14e66b5b{HTTP/1.1}{127.0.0.1:59749}, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=13002, name=searcherExecutor-3454-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 51) Thread[id=13985, name=OverseerThreadFactory-3955-thread-3, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=15405, name=Thread-10248, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 53) Thread[id=15254, name=Thread-10108, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 54) Thread[id=12413, name=Scheduler-1477714105, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=12129, name=searcherExecutor-2924-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 56) Thread[id=15365, name=Thread-10209, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 57) Thread[id=14780, name=Thread-9652, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 58) Thread[id=15262, name=Thread-10115, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 59) Thread[id=15287, name=Thread-10139, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 60) Thread[id=14721, name=Thread-9593, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 61) Thread[id=14771, name=Thread-9643, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 62) Thread[id=15099, name=Thread-9964, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 63) Thread[id=12538, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 64) Thread[id=13168, name=searcherExecutor-3649-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 65) Thread[id=13609, name=searcherExecutor-3867-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=13090, name=searcherExecutor-3591-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 67) Thread[id=12319, name=searcherExecutor-3097-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 68) Thread[id=12622, name=searcherExecutor-3280-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 69) Thread[id=13333, name=searcherExecutor-3791-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 70) Thread[id=14739, name=Thread-9611, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 71) Thread[id=13290, name=qtp1631332304-13290, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 72) Thread[id=14726, name=Thread-9598, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 73) Thread[id=15375, name=Thread-10219, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 74) Thread[id=12179, name=searcherExecutor-2999-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 75) Thread[id=15279, name=Thread-10132, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 76) Thread[id=15286, name=Thread-10138, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 77) Thread[id=15317, name=Thread-10164, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 78) Thread[id=14942, name=Thread-9809, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 79) Thread[id=15380, name=Thread-10224, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 80) Thread[id=15282, name=Thread-10135, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 81) Thread[id=15259, name=Thread-10113, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 82) Thread[id=14957, name=Thread-9824, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 83) Thread[id=14684, name=Thread-9558, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 84) Thread[id=15335, name=Thread-10180, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 85) Thread[id=15357, name=Thread-10201, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 86) Thread[id=14775, name=Thread-9647, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 87) Thread[id=13326, name=searcherExecutor-3784-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 88) Thread[id=15385, name=Thread-10229, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 89) Thread[id=13019, name=searcherExecutor-3484-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 90) Thread[id=12966, name=searcherExecutor-3418-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 91) Thread[id=15398, name=Thread-10241, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 92) Thread[id=15325, name=Thread-10171, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 93) Thread[id=13104, name=searcherExecutor-3594-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 94) Thread[id=12339, name=searcherExecutor-3126-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 95) Thread[id=12992, name=searcherExecutor-3419-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 96) Thread[id=12859, name=qtp1631332304-12859, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 97) Thread[id=14913, name=Thread-9780, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 98) Thread[id=15289, name=Thread-10141, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 99) Thread[id=15238, name=Thread-10093, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 100) Thread[id=11937, name=Scheduler-197755805, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 101) Thread[id=15260, name=Thread-10114, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 102) Thread[id=12144, name=searcherExecutor-2935-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 103) Thread[id=14852, name=zkCallback-936-thread-81-processing-n:127.0.0.1:59749_, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.cloud.ZkController$WatcherImpl.process(ZkController.java:2430) at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:269) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 104) Thread[id=15258, name=Thread-10112, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 105) Thread[id=12781, name=qtp493791191-12781, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 106) Thread[id=15080, name=Thread-9946, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 107) Thread[id=14803, name=Thread-9673, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1040) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:302) at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:299) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:917) at org.apache.solr.core.SolrCore$11.run(SolrCore.java:2596) at org.apache.solr.cloud.ZkController$5.run(ZkController.java:2468) 108) Thread[id=13091, name=searcherExecutor-3600-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 109) Thread[id=15256, name=Thread-10110, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:299) at org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:148) at org.apache.solr.core.SolrConfig.refreshRequestParams(S [...truncated too long message...] le.stop(ContainerLifeCycle.java:143) [junit4] 2> at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:162) [junit4] 2> at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73) [junit4] 2> at org.eclipse.jetty.server.Server.doStop(Server.java:456) [junit4] 2> ... 40 more [junit4] 2> Caused by: java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) [junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) [junit4] 2> at org.eclipse.jetty.server.AbstractConnector.doStop(AbstractConnector.java:298) [junit4] 2> at org.eclipse.jetty.server.AbstractNetworkConnector.doStop(AbstractNetworkConnector.java:88) [junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89) [junit4] 2> at org.eclipse.jetty.server.Server.doStop(Server.java:445) [junit4] 2> ... 40 more [junit4] 2> 9553028 INFO (SUITE-TestReplicationHandler-seed#[745A041AC46B65F1]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_745A041AC46B65F1-001 [junit4] 2> Oct 25, 2015 9:03:36 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll [junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated: [junit4] 2> 1) Thread[id=67450, name=qtp1396019072-67450-selector-ServerConnectorManager@5f87c55d/0, state=RUNNABLE, group=TGRP-TestReplicationHandler] [junit4] 2> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) [junit4] 2> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) [junit4] 2> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) [junit4] 2> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] 2> at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] 2> at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] 2> at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 2) Thread[id=67451, name=qtp1396019072-67451-selector-ServerConnectorManager@5f87c55d/1, state=RUNNABLE, group=TGRP-TestReplicationHandler] [junit4] 2> at sun.nio.ch.IOUtil.drain(Native Method) [junit4] 2> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) [junit4] 2> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] 2> at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] 2> at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] 2> at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 3) Thread[id=67467, name=Scheduler-872252599, state=WAITING, group=TGRP-TestReplicationHandler] [junit4] 2> at sun.misc.Unsafe.park(Native Method) [junit4] 2> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) [junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> NOTE: test params are: codec=Asserting(Lucene53), sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ar_YE, timezone=Indian/Maldives [junit4] 2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_45 (64-bit)/cpus=4,threads=4,free=135742248,total=468189184 [junit4] 2> NOTE: All tests run in this JVM: [CollectionStateFormat2Test, TestSolrXml, FileBasedSpellCheckerTest, TestSolrCloudWithKerberosAlt, HttpPartitionTest, EchoParamsTest, TestLuceneMatchVersion, TestStressLucene, MigrateRouteKeyTest, DateMathParserTest, TestPseudoReturnFields, CursorPagingTest, EnumFieldTest, TestDocBasedVersionConstraints, TestHighlightDedupGrouping, DateRangeFieldTest, TestDocumentBuilder, SparseHLLTest, TestOmitPositions, TestBinaryField, AssignTest, TestConfigSetsAPIZkFailure, TestValueSourceCache, TestDynamicFieldResource, TestFilteredDocIdSet, HdfsThreadLeakTest, DistribJoinFromCollectionTest, TestExactStatsCache, AlternateDirectoryTest, TestPKIAuthenticationPlugin, TestStandardQParsers, TestDistributedSearch, UUIDUpdateProcessorFallbackTest, TestRequestStatusCollectionAPI, FileUtilsTest, SynonymTokenizerTest, DefaultValueUpdateProcessorTest, DistributedQueryComponentOptimizationTest, LeaderInitiatedRecoveryOnCommitTest, AliasIntegrationTest, LeaderElectionIntegrationTest, TestRangeQuery, SmileWriterTest, TestLMDirichletSimilarityFactory, VersionInfoTest, TestSolrConfigHandlerConcurrent, TestExpandComponent, SolrXmlInZkTest, SyncSliceTest, TestCSVLoader, HdfsChaosMonkeySafeLeaderTest, CacheHeaderTest, DateFieldTest, TestCloudManagedSchema, TestFunctionQuery, PrimUtilsTest, OutputWriterTest, TestRestManager, BlockDirectoryTest, BufferStoreTest, TestSimpleQParserPlugin, TestCollectionAPI, LeaderInitiatedRecoveryOnShardRestartTest, BigEndianAscendingWordSerializerTest, OverseerRolesTest, TestArbitraryIndexDir, TestIBSimilarityFactory, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, DataDrivenBlockJoinTest, TestCollationFieldDocValues, RemoteQueryErrorTest, TestExceedMaxTermLength, QueryEqualityTest, CoreMergeIndexesAdminHandlerTest, TestPivotHelperCode, CollectionsAPIAsyncDistributedZkTest, TestSolrConfigHandlerCloud, TestNoOpRegenerator, BasicDistributedZkTest, SpellingQueryConverterTest, TestHighFrequencyDictionaryFactory, SuggestComponentContextFilterQueryTest, ParsingFieldUpdateProcessorsTest, TestRandomFaceting, TestImplicitCoreProperties, TestInfoStreamLogging, SegmentsInfoRequestHandlerTest, JsonLoaderTest, DocumentAnalysisRequestHandlerTest, DeleteInactiveReplicaTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler] [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory. [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestReplicationHandler -Dtests.seed=745A041AC46B65F1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/x1/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_YE -Dtests.timezone=Indian/Maldives -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] ERROR 0.00s J0 | TestReplicationHandler (suite) <<< [junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec). [junit4] > at __randomizedtesting.SeedInfo.seed([745A041AC46B65F1]:0) [junit4] Completed [553/553] on J0 in 7223.33s, 13 tests, 2 errors <<< FAILURES! [...truncated 15 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:782: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:719: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:59: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:233: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:516: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1432: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:987: There were test failures: 553 suites (8 ignored), 2173 tests, 3 suite-level errors, 3 errors, 423 ignored (14 assumptions) [seed: 745A041AC46B65F1] Total time: 296 minutes 34 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts No prior successful build to compare, so performing full copy of artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org