Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/493/ Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
4 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:50128 within 30000 ms Stack Trace: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:50128 within 30000 ms at __randomizedtesting.SeedInfo.seed([6171793A663808C]:0) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98) at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:242) at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:236) at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:121) at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.startup(TestSolrCloudWithDelegationTokens.java:72) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:50128 within 30000 ms at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174) ... 31 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: Stack Trace: java.lang.NullPointerException at __randomizedtesting.SeedInfo.seed([6171793A663808C]:0) at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.shutdown(TestSolrCloudWithDelegationTokens.java:89) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: 27 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens: 1) Thread[id=23954, name=qtp39109339-23954-acceptor-0@2f35613-ServerConnector@b92fc9a{HTTP/1.1,[http/1.1]}{127.0.0.1:35593}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=23941, name=qtp654560265-23941, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=23956, name=org.eclipse.jetty.server.session.HashSessionManager@7c46e0dcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=23946, name=Scheduler-1945824116, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=23961, name=Scheduler-700720793, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=23950, name=org.eclipse.jetty.server.session.HashSessionManager@51fbc78Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=23955, name=Scheduler-1858994604, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=23951, name=qtp39109339-23951, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=23949, name=qtp39109339-23949, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=23948, name=qtp39109339-23948, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=23931, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 12) Thread[id=23944, name=qtp654560265-23944, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=23942, name=qtp654560265-23942, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=23928, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=23938, name=qtp654560265-23938-acceptor-0@1b63445d-ServerConnector@c9f32ec{HTTP/1.1,[http/1.1]}{127.0.0.1:50128}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=23943, name=qtp654560265-23943, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=23947, name=qtp39109339-23947, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=23952, name=qtp39109339-23952, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=23945, name=qtp654560265-23945, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=23953, name=qtp39109339-23953, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=23937, name=qtp654560265-23937, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=23927, name=Thread-8335, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 23) Thread[id=23935, name=jetty-launcher-3432-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=23940, name=qtp654560265-23940, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=23939, name=qtp39109339-23939, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=23936, name=jetty-launcher-3432-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=23929, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 27 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens: 1) Thread[id=23954, name=qtp39109339-23954-acceptor-0@2f35613-ServerConnector@b92fc9a{HTTP/1.1,[http/1.1]}{127.0.0.1:35593}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=23941, name=qtp654560265-23941, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=23956, name=org.eclipse.jetty.server.session.HashSessionManager@7c46e0dcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=23946, name=Scheduler-1945824116, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=23961, name=Scheduler-700720793, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=23950, name=org.eclipse.jetty.server.session.HashSessionManager@51fbc78Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=23955, name=Scheduler-1858994604, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=23951, name=qtp39109339-23951, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=23949, name=qtp39109339-23949, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=23948, name=qtp39109339-23948, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=23931, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 12) Thread[id=23944, name=qtp654560265-23944, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=23942, name=qtp654560265-23942, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=23928, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=23938, name=qtp654560265-23938-acceptor-0@1b63445d-ServerConnector@c9f32ec{HTTP/1.1,[http/1.1]}{127.0.0.1:50128}, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=23943, name=qtp654560265-23943, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=23947, name=qtp39109339-23947, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=23952, name=qtp39109339-23952, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=23945, name=qtp654560265-23945, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=23953, name=qtp39109339-23953, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=23937, name=qtp654560265-23937, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=23927, name=Thread-8335, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 23) Thread[id=23935, name=jetty-launcher-3432-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=23940, name=qtp654560265-23940, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=23939, name=qtp39109339-23939, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=23936, name=jetty-launcher-3432-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=23929, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) at __randomizedtesting.SeedInfo.seed([6171793A663808C]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithDelegationTokens Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=23928, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=23937, name=qtp654560265-23937, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=23935, name=jetty-launcher-3432-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=23956, name=org.eclipse.jetty.server.session.HashSessionManager@7c46e0dcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=23939, name=qtp39109339-23939, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300) at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=23946, name=Scheduler-1945824116, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=23961, name=Scheduler-700720793, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=23936, name=jetty-launcher-3432-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=23950, name=org.eclipse.jetty.server.session.HashSessionManager@51fbc78Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=23955, name=Scheduler-1858994604, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=23928, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=23937, name=qtp654560265-23937, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=23935, name=jetty-launcher-3432-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=23956, name=org.eclipse.jetty.server.session.HashSessionManager@7c46e0dcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=23939, name=qtp39109339-23939, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300) at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=23946, name=Scheduler-1945824116, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=23961, name=Scheduler-700720793, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=23936, name=jetty-launcher-3432-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=23950, name=org.eclipse.jetty.server.session.HashSessionManager@51fbc78Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=23955, name=Scheduler-1858994604, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([6171793A663808C]:0) Build Log: [...truncated 12284 lines...] [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens [junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6171793A663808C-001/init-core-data-001 [junit4] 2> 2264786 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[6171793A663808C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None) [junit4] 2> 2264789 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[6171793A663808C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6171793A663808C-001/tempDir-001 [junit4] 2> 2264790 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[6171793A663808C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 2264790 INFO (Thread-8335) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 2264790 INFO (Thread-8335) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 2264894 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[6171793A663808C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:50128 [junit4] 2> 2264900 INFO (jetty-launcher-3432-thread-1) [ ] o.e.j.s.Server jetty-9.3.8.v20160314 [junit4] 2> 2264900 INFO (jetty-launcher-3432-thread-2) [ ] o.e.j.s.Server jetty-9.3.8.v20160314 [junit4] 2> 2264903 INFO (jetty-launcher-3432-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@29886f3c{/solr,null,AVAILABLE} [junit4] 2> 2264903 INFO (jetty-launcher-3432-thread-1) [ ] o.e.j.s.ServerConnector Started ServerConnector@c9f32ec{HTTP/1.1,[http/1.1]}{127.0.0.1:50128} [junit4] 2> 2264903 INFO (jetty-launcher-3432-thread-1) [ ] o.e.j.s.Server Started @2268553ms [junit4] 2> 2264903 INFO (jetty-launcher-3432-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d178cf3{/solr,null,AVAILABLE} [junit4] 2> 2264903 INFO (jetty-launcher-3432-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50128} [junit4] 2> 2264904 INFO (jetty-launcher-3432-thread-2) [ ] o.e.j.s.ServerConnector Started ServerConnector@b92fc9a{HTTP/1.1,[http/1.1]}{127.0.0.1:35593} [junit4] 2> 2264904 ERROR (jetty-launcher-3432-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2264904 INFO (jetty-launcher-3432-thread-2) [ ] o.e.j.s.Server Started @2268553ms [junit4] 2> 2264905 INFO (jetty-launcher-3432-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35593} [junit4] 2> 2264904 INFO (jetty-launcher-3432-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0 [junit4] 2> 2264905 INFO (jetty-launcher-3432-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2264905 INFO (jetty-launcher-3432-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 2264905 ERROR (jetty-launcher-3432-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 2264905 INFO (jetty-launcher-3432-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-16T21:26:54.608Z [junit4] 2> 2264906 INFO (jetty-launcher-3432-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0 [junit4] 2> 2264906 INFO (jetty-launcher-3432-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 2264906 INFO (jetty-launcher-3432-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 2264906 INFO (jetty-launcher-3432-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-16T21:26:54.609Z [junit4] 2> 2264908 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3f51a51b[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2264908 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3305a9c0[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2264908 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@31965c81{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2264908 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@3dc241d0{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2264908 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2264909 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2266507 WARN (qtp654560265-23945) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@4fc24751[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2266508 WARN (qtp654560265-23945) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@4105b0d9{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2266508 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2266974 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3f51a51b[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2266975 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@39fdc2b0{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2266975 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2268118 WARN (qtp654560265-23943) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@163c7ea6[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2268118 WARN (qtp654560265-23943) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@533ee205{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2268118 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2268511 WARN (qtp654560265-23937) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3305a9c0[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2268511 WARN (qtp654560265-23937) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@34a48a72{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2268512 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2269664 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@4fc24751[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2269664 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@42604cc9{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2269664 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2269813 WARN (qtp654560265-23943) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3f51a51b[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2269813 WARN (qtp654560265-23943) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@171bfb66{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2269814 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2270874 WARN (qtp654560265-23944) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@163c7ea6[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2270874 WARN (qtp654560265-23944) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@91ae409{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2270874 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2271791 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3305a9c0[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2271791 WARN (qtp654560265-23942) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@1cd0b4a4{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2271791 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2272024 WARN (qtp654560265-23937) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@4fc24751[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2272025 WARN (qtp654560265-23937) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@38aa8309{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2272025 WARN (jetty-launcher-3432-thread-1-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and attempting reconnect [junit4] 2> java.io.IOException: Packet len1213486160 is out of range! [junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79) [junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366) [junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) [junit4] 2> 2272917 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@3f51a51b[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>h: 0\r\nConnection:...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} [junit4] 2> 2272917 WARN (qtp654560265-23941) [ ] o.e.j.h.HttpParser bad HTTP parsed: 400 Illegal character 0x0 for HttpChannelOverHttp@41116250{r=0,c=false,a=IDLE,uri=null} [junit4] 2> 2272917 WARN (jetty-launcher-3432-thread-2-SendThread(127.0.0.1:50128)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:50128, unexpected error, closing socket connection and [...truncated too long message...] cuteProduceConsume.java:156) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 3) Thread[id=23935, name=jetty-launcher-3432-thread-1, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 4) Thread[id=23956, name=org.eclipse.jetty.server.session.HashSessionManager@7c46e0dcTimer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 5) Thread[id=23939, name=qtp39109339-23939, state=RUNNABLE, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method) [junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300) [junit4] > at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207) [junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) [junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) [junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) [junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) [junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) [junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) [junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) [junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102) [junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) [junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 6) Thread[id=23946, name=Scheduler-1945824116, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 7) Thread[id=23961, name=Scheduler-700720793, state=WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 8) Thread[id=23936, name=jetty-launcher-3432-thread-2, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) [junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) [junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 9) Thread[id=23950, name=org.eclipse.jetty.server.session.HashSessionManager@51fbc78Timer, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > 10) Thread[id=23955, name=Scheduler-1858994604, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithDelegationTokens] [junit4] > at sun.misc.Unsafe.park(Native Method) [junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) [junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) [junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) [junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > at __randomizedtesting.SeedInfo.seed([6171793A663808C]:0) [junit4] Completed [521/653 (1!)] on J1 in 85.33s, 0 tests, 4 errors <<< FAILURES! [...truncated 61152 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org