Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/13339/ Java: 64bit/jdk1.9.0-ea-b60 -XX:-UseCompressedOops -XX:+UseG1GC
2 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudCluster Error Message: 20 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudCluster: 1) Thread[id=79, name=qtp273890950-79-acceptor-0@61cfea17-ServerConnector@10f91751{HTTP/1.1}{127.0.0.1:35770}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=135, name=jetty-launcher-3-thread-3-SendThread(127.0.0.1:51868), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 3) Thread[id=145, name=Thread-55, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:508) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1095) 4) Thread[id=84, name=qtp273890950-84, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=95, name=org.eclipse.jetty.server.session.HashSessionManager@6ca54d3dTimer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=176, name=zkCallback-23-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=141, name=zkCallback-23-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=169, name=OverseerCollectionProcessor-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350) at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:536) at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:293) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=82, name=qtp273890950-82, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=72, name=qtp273890950-72-selector-ServerConnectorManager@37ceacdf/3, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=168, name=OverseerStateUpdate-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.DistributedQueue$LatchWatcher.await(DistributedQueue.java:276) at org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:320) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:594) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:572) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:192) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=177, name=zkCallback-23-thread-4, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=63, name=qtp273890950-63-selector-ServerConnectorManager@37ceacdf/1, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=170, name=OverseerHdfsCoreFailoverThread-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=53, name=qtp273890950-53-selector-ServerConnectorManager@37ceacdf/0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=91, name=Scheduler-258835486, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=67, name=qtp273890950-67-selector-ServerConnectorManager@37ceacdf/2, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=166, name=zkCallback-23-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=136, name=jetty-launcher-3-thread-3-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 20) Thread[id=87, name=qtp273890950-87, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudCluster: 1) Thread[id=79, name=qtp273890950-79-acceptor-0@61cfea17-ServerConnector@10f91751{HTTP/1.1}{127.0.0.1:35770}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=135, name=jetty-launcher-3-thread-3-SendThread(127.0.0.1:51868), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 3) Thread[id=145, name=Thread-55, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:508) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1095) 4) Thread[id=84, name=qtp273890950-84, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=95, name=org.eclipse.jetty.server.session.HashSessionManager@6ca54d3dTimer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=176, name=zkCallback-23-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=141, name=zkCallback-23-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=169, name=OverseerCollectionProcessor-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350) at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:536) at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:293) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=82, name=qtp273890950-82, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=72, name=qtp273890950-72-selector-ServerConnectorManager@37ceacdf/3, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=168, name=OverseerStateUpdate-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.DistributedQueue$LatchWatcher.await(DistributedQueue.java:276) at org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:320) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:594) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:572) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:192) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=177, name=zkCallback-23-thread-4, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=63, name=qtp273890950-63-selector-ServerConnectorManager@37ceacdf/1, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=170, name=OverseerHdfsCoreFailoverThread-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=53, name=qtp273890950-53-selector-ServerConnectorManager@37ceacdf/0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=91, name=Scheduler-258835486, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=67, name=qtp273890950-67-selector-ServerConnectorManager@37ceacdf/2, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=166, name=zkCallback-23-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=136, name=jetty-launcher-3-thread-3-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 20) Thread[id=87, name=qtp273890950-87, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([CEEE7E44973C0F07]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudCluster Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=135, name=jetty-launcher-3-thread-3-SendThread(127.0.0.1:51868), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 2) Thread[id=145, name=Thread-55, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:508) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1095) 3) Thread[id=95, name=org.eclipse.jetty.server.session.HashSessionManager@6ca54d3dTimer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=176, name=zkCallback-23-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=141, name=zkCallback-23-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=72, name=qtp273890950-72-selector-ServerConnectorManager@37ceacdf/3, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=177, name=zkCallback-23-thread-4, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=63, name=qtp273890950-63-selector-ServerConnectorManager@37ceacdf/1, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=170, name=OverseerHdfsCoreFailoverThread-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=53, name=qtp273890950-53-selector-ServerConnectorManager@37ceacdf/0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=91, name=Scheduler-258835486, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=67, name=qtp273890950-67-selector-ServerConnectorManager@37ceacdf/2, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=166, name=zkCallback-23-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=135, name=jetty-launcher-3-thread-3-SendThread(127.0.0.1:51868), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 2) Thread[id=145, name=Thread-55, state=WAITING, group=TGRP-TestMiniSolrCloudCluster] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:508) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1095) 3) Thread[id=95, name=org.eclipse.jetty.server.session.HashSessionManager@6ca54d3dTimer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=176, name=zkCallback-23-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=141, name=zkCallback-23-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=72, name=qtp273890950-72-selector-ServerConnectorManager@37ceacdf/3, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=177, name=zkCallback-23-thread-4, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=63, name=qtp273890950-63-selector-ServerConnectorManager@37ceacdf/1, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=170, name=OverseerHdfsCoreFailoverThread-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=53, name=qtp273890950-53-selector-ServerConnectorManager@37ceacdf/0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=91, name=Scheduler-258835486, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=67, name=qtp273890950-67-selector-ServerConnectorManager@37ceacdf/2, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=166, name=zkCallback-23-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([CEEE7E44973C0F07]:0) Build Log: [...truncated 9581 lines...] [junit4] Suite: org.apache.solr.cloud.TestMiniSolrCloudCluster [junit4] 2> 6184 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 6187 INFO (Thread-7) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 6187 INFO (Thread-7) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 6286 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.ZkTestServer start zk server on port:51868 [junit4] 2> 6300 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6333 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6395 INFO (zkCallback-4-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7afb054a name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6395 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6396 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6405 INFO (TEST-TestMiniSolrCloudCluster.testExraFilters-seed#[CEEE7E44973C0F07]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml [junit4] 2> 6555 INFO (jetty-launcher-3-thread-2) [ ] o.e.j.s.Server jetty-9.2.11.v20150529 [junit4] 2> 6556 INFO (jetty-launcher-3-thread-3) [ ] o.e.j.s.Server jetty-9.2.11.v20150529 [junit4] 2> 6555 INFO (jetty-launcher-3-thread-1) [ ] o.e.j.s.Server jetty-9.2.11.v20150529 [junit4] 2> 6556 INFO (jetty-launcher-3-thread-4) [ ] o.e.j.s.Server jetty-9.2.11.v20150529 [junit4] 2> 6555 INFO (jetty-launcher-3-thread-5) [ ] o.e.j.s.Server jetty-9.2.11.v20150529 [junit4] 2> 6640 INFO (jetty-launcher-3-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@153dcb83{/solr,null,AVAILABLE} [junit4] 2> 6640 INFO (jetty-launcher-3-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7e9febc9{/solr,null,AVAILABLE} [junit4] 2> 6640 INFO (jetty-launcher-3-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ee5e922{/solr,null,AVAILABLE} [junit4] 2> 6640 INFO (jetty-launcher-3-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4732963c{/solr,null,AVAILABLE} [junit4] 2> 6646 INFO (jetty-launcher-3-thread-5) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61ddb7ae{/solr,null,AVAILABLE} [junit4] 2> 6654 INFO (jetty-launcher-3-thread-2) [ ] o.e.j.s.ServerConnector Started ServerConnector@685d726{HTTP/1.1}{127.0.0.1:37809} [junit4] 2> 6655 INFO (jetty-launcher-3-thread-2) [ ] o.e.j.s.Server Started @8070ms [junit4] 2> 6655 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37809} [junit4] 2> 6655 INFO (jetty-launcher-3-thread-4) [ ] o.e.j.s.ServerConnector Started ServerConnector@6be24824{HTTP/1.1}{127.0.0.1:47913} [junit4] 2> 6655 INFO (jetty-launcher-3-thread-1) [ ] o.e.j.s.ServerConnector Started ServerConnector@163a548b{HTTP/1.1}{127.0.0.1:46392} [junit4] 2> 6657 INFO (jetty-launcher-3-thread-4) [ ] o.e.j.s.Server Started @8072ms [junit4] 2> 6662 INFO (jetty-launcher-3-thread-1) [ ] o.e.j.s.Server Started @8073ms [junit4] 2> 6655 INFO (jetty-launcher-3-thread-5) [ ] o.e.j.s.ServerConnector Started ServerConnector@43e7fdaa{HTTP/1.1}{127.0.0.1:45040} [junit4] 2> 6654 INFO (jetty-launcher-3-thread-3) [ ] o.e.j.s.ServerConnector Started ServerConnector@10f91751{HTTP/1.1}{127.0.0.1:35770} [junit4] 2> 6670 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@45fe3ee3 [junit4] 2> 6671 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/' [junit4] 2> 6662 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46392} [junit4] 2> 6658 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47913} [junit4] 2> 6665 INFO (jetty-launcher-3-thread-5) [ ] o.e.j.s.Server Started @8080ms [junit4] 2> 6671 INFO (jetty-launcher-3-thread-3) [ ] o.e.j.s.Server Started @8086ms [junit4] 2> 6674 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45040} [junit4] 2> 6674 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@45fe3ee3 [junit4] 2> 6678 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/' [junit4] 2> 6679 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@45fe3ee3 [junit4] 2> 6680 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/' [junit4] 2> 6674 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35770} [junit4] 2> 6679 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@45fe3ee3 [junit4] 2> 6683 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/' [junit4] 2> 6684 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@45fe3ee3 [junit4] 2> 6685 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/' [junit4] 2> 6711 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.s.SolrDispatchFilter Trying to read solr.xml from 127.0.0.1:51868/solr [junit4] 2> 6711 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6712 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6716 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.s.SolrDispatchFilter Trying to read solr.xml from 127.0.0.1:51868/solr [junit4] 2> 6716 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6716 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6719 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.s.SolrDispatchFilter Trying to read solr.xml from 127.0.0.1:51868/solr [junit4] 2> 6719 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6720 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6721 INFO (zkCallback-5-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@548fedc2 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6723 INFO (zkCallback-6-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@340c6813 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6723 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6724 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6725 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6726 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6738 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.s.SolrDispatchFilter Trying to read solr.xml from 127.0.0.1:51868/solr [junit4] 2> 6739 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.s.SolrDispatchFilter Trying to read solr.xml from 127.0.0.1:51868/solr [junit4] 2> 6740 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6740 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6739 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6740 INFO (zkCallback-7-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@d903821 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6741 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6743 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6751 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6752 INFO (zkCallback-9-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3281bd98 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6753 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6753 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6756 INFO (zkCallback-8-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@50bcf77a name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6756 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6795 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6808 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.CoresLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/. [junit4] 2> 6810 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.CoreContainer New CoreContainer 1187455624 [junit4] 2> 6811 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/] [junit4] 2> 6811 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.CoreContainer loading shared library: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib [junit4] 2> 6813 WARN (jetty-launcher-3-thread-4) [ ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib). [junit4] 2> 6815 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.CoresLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/. [junit4] 2> 6817 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x14e606226dc0001, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 6816 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.CoresLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/. [junit4] 2> 6816 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.CoresLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/. [junit4] 2> 6820 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.CoreContainer New CoreContainer 1169836023 [junit4] 2> 6817 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.CoreContainer New CoreContainer 777426133 [junit4] 2> 6820 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/] [junit4] 2> 6820 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.CoreContainer loading shared library: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib [junit4] 2> 6820 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/] [junit4] 2> 6820 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.CoreContainer New CoreContainer 401430148 [junit4] 2> 6820 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.CoreContainer loading shared library: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib [junit4] 2> 6820 WARN (jetty-launcher-3-thread-2) [ ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib). [junit4] 2> 6830 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.CoresLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/. [junit4] 2> 6831 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.CoreContainer New CoreContainer 1273338310 [junit4] 2> 6831 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/] [junit4] 2> 6831 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.CoreContainer loading shared library: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib [junit4] 2> 6831 WARN (jetty-launcher-3-thread-3) [ ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib). [junit4] 2> 6830 WARN (jetty-launcher-3-thread-5) [ ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib). [junit4] 2> 6827 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false, [junit4] 2> 6835 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 6835 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 6835 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 6836 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51868/solr [junit4] 2> 6821 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/] [junit4] 2> 6837 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.CoreContainer loading shared library: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib [junit4] 2> 6837 WARN (jetty-launcher-3-thread-1) [ ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestMiniSolrCloudCluster_CEEE7E44973C0F07-001/tempDir-001/lib). [junit4] 2> 6849 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false, [junit4] 2> 6852 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false, [junit4] 2> 6852 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 6853 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 6853 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 6853 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false, [junit4] 2> 6853 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 6853 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 6853 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 6853 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51868/solr [junit4] 2> 6854 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51868/solr [junit4] 2> 6854 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 6855 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false, [junit4] 2> 6855 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 6856 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 6856 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51868/solr [junit4] 2> 6856 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 6857 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 6857 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 6857 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51868/solr [junit4] 2> 6864 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 6864 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 6864 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 6864 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 6867 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6864 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 6866 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6866 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6866 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6869 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6867 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6867 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 6870 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6872 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6874 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6876 INFO (zkCallback-18-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@287be2a0 name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6876 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6876 INFO (jetty-launcher-3-thread-5) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6877 INFO (zkCallback-17-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@b28c5c0 name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6877 INFO (zkCallback-16-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52679bf5 name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6877 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6877 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6877 INFO (jetty-launcher-3-thread-2) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6878 INFO (jetty-launcher-3-thread-1) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6878 INFO (zkCallback-19-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@588b6f42 name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6878 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6879 INFO (jetty-launcher-3-thread-3) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6880 INFO (zkCallback-15-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@163f576 name:ZooKeeperConnection Watcher:127.0.0.1:51868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6880 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6880 INFO (jetty-launcher-3-thread-4) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 6886 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6889 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6888 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6890 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6890 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 6892 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:45040_solr) [n:127.0.0.1:45040_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@10ed3e9c name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6892 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6894 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:46392_solr) [n:127.0.0.1:46392_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7632a8cd name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6894 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6895 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:35770_solr) [n:127.0.0.1:35770_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7350d29 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6895 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6896 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:37809_solr) [n:127.0.0.1:37809_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@453abb85 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6896 INFO (zkCallback-24-thread-1-processing-n:127.0.0.1:47913_solr) [n:127.0.0.1:47913_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@23a984a3 name:ZooKeeperConnection Watcher:127.0.0.1:51868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 6896 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6897 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 6913 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 6914 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 6914 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 6914 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 6915 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 6941 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 6943 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 6943 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 6944 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 6944 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 6953 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 6954 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 6954 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 6955 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 6956 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 6965 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 6965 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 6967 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 6967 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 6976 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 6978 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 6978 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 6978 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 6978 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 6995 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 6995 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 6995 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 6996 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 6996 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 7002 INFO (jetty-launcher-3-thread-5) [n:127.0.0.1:45040_solr ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 7002 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 7003 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 7003 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 7003 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 7008 INFO (jetty-launcher-3-thread-3) [n:127.0.0.1:35770_solr ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 7009 INFO (jetty-launcher-3-thread-2) [n:127.0.0.1:37809_solr ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 7009 INFO (jetty-launcher-3-thread-1) [n:127.0.0.1:46392_solr ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 7009 INFO (jetty-launcher-3-thread-4) [n:127.0.0.1:47913_solr ] o.a.s.c.c.S [...truncated too long message...] mpl.doSelect(EPollSelectorImpl.java:90) [junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] > at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 7) Thread[id=177, name=zkCallback-23-thread-4, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 8) Thread[id=63, name=qtp273890950-63-selector-ServerConnectorManager@37ceacdf/1, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.nio.ch.IOUtil.drain(Native Method) [junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) [junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] > at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 9) Thread[id=170, name=OverseerHdfsCoreFailoverThread-94118616896372751-127.0.0.1:35770_solr-n_0000000002, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] [junit4] > at java.lang.Thread.sleep(Native Method) [junit4] > at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 10) Thread[id=53, name=qtp273890950-53-selector-ServerConnectorManager@37ceacdf/0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) [junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) [junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193) [junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) [junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) [junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) [junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] > at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 11) Thread[id=91, name=Scheduler-258835486, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 12) Thread[id=67, name=qtp273890950-67-selector-ServerConnectorManager@37ceacdf/2, state=RUNNABLE, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) [junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) [junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) [junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) [junit4] > at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) [junit4] > at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 13) Thread[id=166, name=zkCallback-23-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudCluster] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > at __randomizedtesting.SeedInfo.seed([CEEE7E44973C0F07]:0) [junit4] Completed [13/514] on J1 in 64.97s, 4 tests, 2 errors <<< FAILURES! [...truncated 1514 lines...] BUILD FAILED /home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:526: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:474: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:230: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:513: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1426: The following error occurred while executing this line: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:981: There were test failures: 514 suites, 2022 tests, 2 suite-level errors, 1097 ignored (18 assumptions) Total time: 29 minutes 23 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org