[JENKINS] Lucene-Solr-5.x-Linux (32bit/jdk1.8.0_60) - Build # 13920 - Failure!

[Policeman Jenkins Server]

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-5.x-Linux/13920/
Java: 32bit/jdk1.8.0_60 -client -XX:+UseConcMarkSweepGC

29 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
19 threads leaked from SUITE scope at 
org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=295, 
name=OverseerCollectionConfigSetProcessor-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)    
     at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
         at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)    
     at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)  
       at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:338)   
      at 
org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:210)    
     at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:197) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=317, 
name=OverseerStateUpdate-94513301757886467-127.0.0.1:7000_solr-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)    
     at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
         at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)    
     at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)  
       at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:166)         
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:144)       
  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)  
       at java.lang.Thread.run(Thread.java:745)    3) Thread[id=315, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-EventThread,
 state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    
4) Thread[id=321, name=zkCallback-28-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=318, 
name=OverseerCollectionConfigSetProcessor-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)    
     at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
         at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)    
     at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)  
       at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:338)   
      at 
org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:210)    
     at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:197) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=303, 
name=zkCallback-23-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=316, 
name=zkCallback-28-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    8) Thread[id=304, 
name=zkCallback-23-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=322, 
name=zkCallback-28-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=292, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-EventThread,
 state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
11) Thread[id=296, 
name=OverseerHdfsCoreFailoverThread-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)         
at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:730)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=314, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:52562),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   
13) Thread[id=297, name=Thread-173, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)   14) 
Thread[id=293, name=zkCallback-23-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   15) Thread[id=294, 
name=OverseerStateUpdate-94513301652504579-127.0.0.1:7000_solr-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)    
     at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
         at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)    
     at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)  
       at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:166)         
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:144)       
  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)  
       at java.lang.Thread.run(Thread.java:745)   16) Thread[id=291, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:33137),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   17) 
Thread[id=323, name=zkCallback-28-thread-4, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   18) Thread[id=319, 
name=OverseerHdfsCoreFailoverThread-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)   
      at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
         at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)         
at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:730)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=320, 
name=Thread-175, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]       
  at java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 19 threads leaked from 
SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=295, 
name=OverseerCollectionConfigSetProcessor-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
        at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
        at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)
        at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)
        at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:338)
        at 
org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:210)
        at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:197)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=317, 
name=OverseerStateUpdate-94513301757886467-127.0.0.1:7000_solr-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
        at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
        at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)
        at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:166)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:144)
        at 
org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=315, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-EventThread,
 state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   4) Thread[id=321, name=zkCallback-28-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=318, 
name=OverseerCollectionConfigSetProcessor-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
        at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
        at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)
        at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)
        at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:338)
        at 
org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:210)
        at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:197)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=303, name=zkCallback-23-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=316, name=zkCallback-28-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=304, name=zkCallback-23-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=322, name=zkCallback-28-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=292, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-EventThread,
 state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  11) Thread[id=296, 
name=OverseerHdfsCoreFailoverThread-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)
        at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:730)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=314, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:52562),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  13) Thread[id=297, name=Thread-173, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)
  14) Thread[id=293, name=zkCallback-23-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=294, 
name=OverseerStateUpdate-94513301652504579-127.0.0.1:7000_solr-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
        at 
org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:311)
        at 
org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:290)
        at 
org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:359)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:166)
        at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:144)
        at 
org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=291, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:33137),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  17) Thread[id=323, name=zkCallback-28-thread-4, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=319, 
name=OverseerHdfsCoreFailoverThread-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:108)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:76)
        at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)
        at 
org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:730)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=320, name=Thread-175, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)
        at __randomizedtesting.SeedInfo.seed([95CCA6E410A8CB11]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=321, name=zkCallback-28-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=303, 
name=zkCallback-23-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=316, 
name=zkCallback-28-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=304, 
name=zkCallback-23-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=322, 
name=zkCallback-28-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=296, 
name=OverseerHdfsCoreFailoverThread-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=314, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:52562),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    8) 
Thread[id=297, name=Thread-173, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)    9) 
Thread[id=293, name=zkCallback-23-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=291, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:33137),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)    
     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   
11) Thread[id=323, name=zkCallback-28-thread-4, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   12) Thread[id=319, 
name=OverseerHdfsCoreFailoverThread-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=320, 
name=Thread-175, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]       
  at java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=321, name=zkCallback-28-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=303, name=zkCallback-23-thread-2, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=316, name=zkCallback-28-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=304, name=zkCallback-23-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=322, name=zkCallback-28-thread-3, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=296, 
name=OverseerHdfsCoreFailoverThread-94513301652504579-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=314, 
name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:52562),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   8) Thread[id=297, name=Thread-173, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)
   9) Thread[id=293, name=zkCallback-23-thread-1, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=291, 
name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:33137),
 state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at 
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  11) Thread[id=323, name=zkCallback-28-thread-4, state=TIMED_WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=319, 
name=OverseerHdfsCoreFailoverThread-94513301757886467-127.0.0.1:7000_solr-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=320, name=Thread-175, state=WAITING, 
group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)
        at __randomizedtesting.SeedInfo.seed([95CCA6E410A8CB11]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
56 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest:     
1) Thread[id=3215, name=Scheduler-8530160, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3273, 
name=Scheduler-20138308, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]        
 at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=3226, 
name=OverseerStateUpdate-94513322291363844-127.0.0.1:39086_-n_0000000000, 
state=TIMED_WAITING, group=Overseer state updater.]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:170)         
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:144)       
  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:193)  
       at java.lang.Thread.run(Thread.java:745)    4) Thread[id=3230, 
name=Thread-1284, state=WAITING, group=TGRP-RecoveryZkTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)    5) 
Thread[id=3228, 
name=OverseerHdfsCoreFailoverThread-94513322291363844-127.0.0.1:39086_-n_0000000000,
 state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at 
java.lang.Thread.sleep(Native Method)         at 
org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=3274, 
name=org.eclipse.jetty.server.session.HashSessionManager@bdba7dTimer, 
state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=3248, 
name=Scheduler-32210562, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]        
 at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    8) Thread[id=3269, 
name=qtp4749150-3269-acceptor-0@1069d34-ServerConnector@14e9b2f{HTTP/1.1}{127.0.0.1:40691},
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=3211, 
name=qtp6873778-3211-acceptor-0@1d44d9b-ServerConnector@113a851{HTTP/1.1}{127.0.0.1:39086},
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=3268, 
name=qtp4749150-3268-selector-ServerConnectorManager@1a3bede/3, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   11) Thread[id=3216, 
name=org.eclipse.jetty.server.session.HashSessionManager@169aee5Timer, 
state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   12) Thread[id=3272, 
name=qtp4749150-3272, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   13) Thread[id=3281, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:36355),
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   14) 
Thread[id=3270, name=qtp4749150-3270, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   15) Thread[id=3195, 
name=Thread-1274, state=WAITING, group=TGRP-RecoveryZkTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Thread.join(Thread.java:1245)         at 
java.lang.Thread.join(Thread.java:1319)         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
         at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)
         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)   
16) Thread[id=3256, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:36355),
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   17) 
Thread[id=3247, name=qtp5622402-3247, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   18) Thread[id=3262, 
name=coreContainerWorkExecutor-1348-thread-1, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   19) Thread[id=3214, 
name=qtp6873778-3214, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   20) Thread[id=3208, 
name=qtp6873778-3208-selector-ServerConnectorManager@c2fe29/1, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   21) Thread[id=3263, 
name=zkCallback-420-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]   
      at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   22) Thread[id=3196, 
name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
         at java.lang.Thread.run(Thread.java:745)   23) Thread[id=3244, 
name=qtp5622402-3244-acceptor-0@b51308-ServerConnector@19a12d2{HTTP/1.1}{127.0.0.1:59063},
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   24) Thread[id=3233, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:36355),
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   25) 
Thread[id=3243, name=qtp5622402-3243-selector-ServerConnectorManager@1117fdc/3, 
state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   26) Thread[id=3249, 
name=org.eclipse.jetty.server.session.HashSessionManager@16ba5e3Timer, 
state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   27) Thread[id=3257, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-EventThread, 
state=WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
28) Thread[id=3267, 
name=qtp4749150-3267-selector-ServerConnectorManager@1a3bede/2, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   29) Thread[id=3240, 
name=qtp5622402-3240-selector-ServerConnectorManager@1117fdc/0, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   30) Thread[id=3199, 
name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]  
       at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   31) Thread[id=3246, name=qtp5622402-3246, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   32) Thread[id=3285, 
name=Thread-1306, state=WAITING, group=TGRP-RecoveryZkTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)   33) 
Thread[id=3227, 
name=OverseerCollectionConfigSetProcessor-94513322291363844-127.0.0.1:39086_-n_0000000000,
 state=TIMED_WAITING, group=Overseer collection creation process.]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:344)   
      at 
org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:210)    
     at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:197) 
        at java.lang.Thread.run(Thread.java:745)   34) Thread[id=3213, 
name=qtp6873778-3213, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   35) Thread[id=3225, 
name=zkCallback-420-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]   
      at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   36) Thread[id=3232, 
name=coreContainerWorkExecutor-1341-thread-1, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   37) Thread[id=3282, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-EventThread, 
state=WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
38) Thread[id=3197, name=SessionTracker, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)      
   at 
org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 
  39) Thread[id=3258, name=zkCallback-427-thread-1, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   40) Thread[id=3271, 
name=qtp4749150-3271, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   41) Thread[id=3260, 
name=Thread-1295, state=WAITING, group=TGRP-RecoveryZkTest]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1153)   42) 
Thread[id=3224, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-EventThread, 
state=WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
43) Thread[id=3287, name=coreContainerWorkExecutor-1354-thread-1, 
state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   44) Thread[id=3234, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-EventThread, 
state=WAITING, group=TGRP-RecoveryZkTest]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   
45) Thread[id=3283, name=zkCallback-431-thread-1, state=TIMED_WAITING, 
group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)       
  at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)     
    at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   46) Thread[id=3265, 
name=qtp4749150-3265-selector-ServerConnectorManager@1a3bede/0, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   47) Thread[id=3212, 
name=qtp6873778-3212, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   48) Thread[id=3245, 
name=qtp5622402-3245, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         
at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) 
        at java.lang.Thread.run(Thread.java:745)   49) Thread[id=3242, 
name=qtp5622402-3242-selector-ServerConnectorManager@1117fdc/2, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   50) Thread[id=3210, 
name=qtp6873778-3210-selector-ServerConnectorManager@c2fe29/3, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   51) Thread[id=3209, 
name=qtp6873778-3209-selector-ServerConnectorManager@c2fe29/2, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   52) Thread[id=3207, 
name=qtp6873778-3207-selector-ServerConnectorManager@c2fe29/0, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   53) Thread[id=3235, 
name=zkCallback-422-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]   
      at sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
         at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
         at 
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         
at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   54) Thread[id=3266, 
name=qtp4749150-3266-selector-ServerConnectorManager@1a3bede/1, state=RUNNABLE, 
group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)   55) Thread[id=3223, 
name=TEST-RecoveryZkTest.test-seed#[95CCA6E410A8CB11]-SendThread(127.0.0.1:36355),
 state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
         at 
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   56) 
Thread[id=3241, name=qtp5622402-3241-selector-ServerConnectorManager@1117fdc/1, 
state=RUNNABLE, group=TGRP-RecoveryZkTest]         at 
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)
         at 
org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)
         at 
org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) 
        at java.lang.Thread.run(Thread.java:745)

Stack Trace:
co

[...truncated too long message...]

er Overseer (id=94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) starting
   [junit4]   2> 757025 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.a.s.c.ChaosMonkey monkey: stop shard! 48664
   [junit4]   2> 757025 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=20039611
   [junit4]   2> 757028 INFO  
(zkCallback-670-thread-2-processing-n:127.0.0.1:48664_eid%2Ft) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.OverseerAutoReplicaFailoverThread 
Starting OverseerAutoReplicaFailoverThread 
autoReplicaFailoverWorkLoopDelay=10000 
autoReplicaFailoverWaitAfterExpiration=30000 
autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 757028 INFO  
(OverseerCollectionConfigSetProcessor-94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004)
 [n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.OverseerTaskProcessor Process current 
queue of overseer operations
   [junit4]   2> 757028 INFO  
(OverseerStateUpdate-94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.Overseer Starting to work on the main 
queue
   [junit4]   2> 757029 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.a.s.c.Overseer Overseer 
(id=94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) closing
   [junit4]   2> 757029 INFO  
(OverseerStateUpdate-94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.Overseer According to ZK I 
(id=94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) am no longer a 
leader.
   [junit4]   2> 757029 ERROR 
(OverseerCollectionConfigSetProcessor-94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004)
 [n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.OverseerTaskProcessor Unable to 
prioritize overseer 
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:68)
   [junit4]   2>        at 
org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:308)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerNodePrioritizer.prioritizeOverseerNodes(OverseerNodePrioritizer.java:59)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:159)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 757030 INFO  
(zkCallback-670-thread-2-processing-n:127.0.0.1:48664_eid%2Ft) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.c.ZkStateReader A live node change: 
WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has 
occurred - updating... (live nodes size: 1)
   [junit4]   2> 757029 INFO  
(OverseerStateUpdate-94513348343693330-127.0.0.1:48664_eid%2Ft-n_0000000004) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:48664_eid%2Ft
   [junit4]   2> 757030 WARN  
(zkCallback-670-thread-2-processing-n:127.0.0.1:48664_eid%2Ft) 
[n:127.0.0.1:48664_eid%2Ft    ] o.a.s.c.c.ZkStateReader ZooKeeper watch 
triggered, but Solr cannot talk to ZK
   [junit4]   2> 757031 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.e.j.s.ServerConnector Stopped ServerConnector@28d959{HTTP/1.1}{127.0.0.1:0}
   [junit4]   2> 757032 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@d88801{/eid/t,null,UNAVAILABLE}
   [junit4]   2> 757033 INFO  
(TEST-TestRebalanceLeaders.test-seed#[95CCA6E410A8CB11]) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:36059 36059
   [junit4]   2> 757070 INFO  (Thread-1855) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:36059 36059
   [junit4]   2> 757071 WARN  (Thread-1855) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/aliases.json
   [junit4]   2>        5       /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/clusterstate.json
   [junit4]   2>        4       /solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        6       /solr/live_nodes
   [junit4]   2>        6       /solr/collections
   [junit4]   2>        2       /solr/overseer/queue
   [junit4]   2>        2       /solr/overseer/collection-queue-work
   [junit4]   2>        2       /solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestRebalanceLeaders -Dtests.method=test 
-Dtests.seed=95CCA6E410A8CB11 -Dtests.multiplier=3 -Dtests.slow=true 
-Dtests.locale=ar_SA -Dtests.timezone=Africa/Asmera -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   24.0s J1 | TestRebalanceLeaders.test <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No 
registered leader was found after waiting for 10000ms , collection: collection1 
slice: shard1
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([95CCA6E410A8CB11:1D98993EBE54A6E9]:0)
   [junit4]    >        at 
org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:642)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:405)
   [junit4]    >        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:310)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:961)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:938)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 757076 INFO  
(SUITE-TestRebalanceLeaders-seed#[95CCA6E410A8CB11]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TestRebalanceLeaders_95CCA6E410A8CB11-001
   [junit4]   2> Sep 13, 2015 3:47:49 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene53): {}, 
docValues:{}, sim=DefaultSimilarity, locale=ar_SA, timezone=Africa/Asmera
   [junit4]   2> NOTE: Linux 3.19.0-28-generic i386/Oracle Corporation 1.8.0_60 
(32-bit)/cpus=12,threads=1,free=67599512,total=241111040
   [junit4]   2> NOTE: All tests run in this JVM: [TestAddFieldRealTimeGet, 
TestPartialUpdateDeduplication, InfoHandlerTest, RulesTest, 
SpellCheckCollatorTest, ZkControllerTest, TestDistributedGrouping, 
TestSolrCLIRunExample, VMParamsZkACLAndCredentialsProvidersTest, 
TestBinaryResponseWriter, TestBulkSchemaAPI, TestLFUCache, HdfsLockFactoryTest, 
ShardRoutingTest, MBeansHandlerTest, OverseerRolesTest, 
TestDistributedStatsComponentCardinality, TestCloudManagedSchema, 
SolrIndexSplitterTest, SimpleMLTQParserTest, 
UniqFieldsUpdateProcessorFactoryTest, 
IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestTrie, 
DirectSolrSpellCheckerTest, TestFiltering, SuggesterFSTTest, 
TestSortingResponseWriter, TestManagedSchemaFieldResource, 
TestStressUserVersions, RecoveryAfterSoftCommitTest, 
TestComplexPhraseQParserPlugin, TestSha256AuthenticationProvider, 
TestReplicaProperties, TestPKIAuthenticationPlugin, MultiTermTest, 
SharedFSAutoReplicaFailoverTest, IndexBasedSpellCheckerTest, 
BasicFunctionalityTest, TestFieldSortValues, CollectionsAPIDistributedZkTest, 
TestQueryUtils, TestIntervalFaceting, TestConfigSets, SolrTestCaseJ4Test, 
TestDistribIDF, SpellPossibilityIteratorTest, TestOrdValues, 
TestSuggestSpellingConverter, DocValuesTest, TestQuerySenderNoQuery, 
TestSerializedLuceneMatchVersion, TestSortByMinMaxFunction, SpatialFilterTest, 
BigEndianAscendingWordSerializerTest, TestSurroundQueryParser, TestUpdate, 
ChaosMonkeySafeLeaderTest, TestSolrDeletionPolicy2, 
WordBreakSolrSpellCheckerTest, CloudMLTQParserTest, 
TestLMJelinekMercerSimilarityFactory, MoreLikeThisHandlerTest, 
CircularListTest, DeleteReplicaTest, NotRequiredUniqueKeyTest, 
ClusterStateUpdateTest, TestCoreContainer, SolrInfoMBeanTest, 
TestBM25SimilarityFactory, TestMacros, RequestLoggingTest, 
SaslZkACLProviderTest, TestCryptoKeys, SchemaVersionSpecificBehaviorTest, 
ExplicitHLLTest, SpatialRPTFieldTypeTest, HttpPartitionTest, 
AnalyticsMergeStrategyTest, SliceStateTest, TestSolrCoreProperties, 
TestCodecSupport, TestNamedUpdateProcessors, 
TestDynamicFieldCollectionResource, TestEmbeddedSolrServerConstructors, 
TestRebalanceLeaders]
   [junit4] Completed [538/538] on J1 in 24.28s, 1 test, 1 error <<< FAILURES!

[...truncated 20 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/build.xml:785: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/build.xml:729: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/build.xml:59: The following error 
occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/solr/build.xml:233: The following 
error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/solr/common-build.xml:524: The 
following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/lucene/common-build.xml:1452: The 
following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-5.x-Linux/lucene/common-build.xml:1006: 
There were test failures: 538 suites, 2127 tests, 4 suite-level errors, 23 
errors, 2 failures, 1520 ignored (26 assumptions)

Total time: 41 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org