Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/33/
11 tests failed. FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test Error Message: Timeout occured while waiting response from server at: http://127.0.0.1:36095/collection1 Stack Trace: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36095/collection1 at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1590) at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:215) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ... 53 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest Error Message: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=11700, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=11698, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=11700, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=11698, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0) FAILED: org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete Error Message: Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> Stack Trace: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> at __randomizedtesting.SeedInfo.seed([27819D609D1F050:A182B7738E391AF5]:0) at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237) at org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete(TestCollectionsAPIViaSolrCloudCluster.java:170) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more FAILED: org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup Error Message: missing cleanup event: [CapturedEvent{timestamp=2072443178755382, stage=STARTED, actionName='null', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443180737590, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443198284716, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/staleShard-splitting={stateTimestamp=1540764857375462512, currentTimeNs=1540937657424373220, deltaSec=172800, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443201434624, stage=SUCCEEDED, actionName='null', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448204734725, stage=STARTED, actionName='null', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448205469030, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448209499488, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448210982445, stage=SUCCEEDED, actionName='null', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453212386765, stage=STARTED, actionName='null', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453213083124, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453215766094, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453217129932, stage=SUCCEEDED, actionName='null', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458218763335, stage=STARTED, actionName='null', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458219652691, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458222208788, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458223619204, stage=SUCCEEDED, actionName='null', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225175175, stage=STARTED, actionName='null', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225893312, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463228827718, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463230727086, stage=SUCCEEDED, actionName='null', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468232425669, stage=STARTED, actionName='null', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468233201881, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468235525545, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/shard1-splitting={stateTimestamp=1540937657422126363, currentTimeNs=1540937682462216654, deltaSec=25, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}] Stack Trace: java.lang.AssertionError: missing cleanup event: [CapturedEvent{timestamp=2072443178755382, stage=STARTED, actionName='null', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443180737590, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443198284716, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/staleShard-splitting={stateTimestamp=1540764857375462512, currentTimeNs=1540937657424373220, deltaSec=172800, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443201434624, stage=SUCCEEDED, actionName='null', event={ "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb", "source":".scheduled_maintenance", "eventTime":2072443175062954, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937657402, "_enqueue_time_":2072443176907937}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448204734725, stage=STARTED, actionName='null', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448205469030, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448209499488, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448210982445, stage=SUCCEEDED, actionName='null', event={ "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh", "source":".scheduled_maintenance", "eventTime":2072448203753643, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937662431, "_enqueue_time_":2072448203980419}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453212386765, stage=STARTED, actionName='null', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453213083124, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453215766094, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453217129932, stage=SUCCEEDED, actionName='null', event={ "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi", "source":".scheduled_maintenance", "eventTime":2072453211478811, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937667439, "_enqueue_time_":2072453211731127}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458218763335, stage=STARTED, actionName='null', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458219652691, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458222208788, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458223619204, stage=SUCCEEDED, actionName='null', event={ "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj", "source":".scheduled_maintenance", "eventTime":2072458217745655, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937672445, "_enqueue_time_":2072458218015497}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225175175, stage=STARTED, actionName='null', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225893312, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463228827718, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463230727086, stage=SUCCEEDED, actionName='null', event={ "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk", "source":".scheduled_maintenance", "eventTime":2072463224201736, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937677451, "_enqueue_time_":2072463224445698}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468232425669, stage=STARTED, actionName='null', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468233201881, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468235525545, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={ "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl", "source":".scheduled_maintenance", "eventTime":2072468231314810, "eventType":"SCHEDULED", "properties":{ "actualEventTime":1540937682458, "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/shard1-splitting={stateTimestamp=1540937657422126363, currentTimeNs=1540937682462216654, deltaSec=25, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={ "trigger":".scheduled_maintenance", "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED"], "beforeAction":"inactive_shard_plan", "afterAction":"inactive_shard_plan", "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}] at __randomizedtesting.SeedInfo.seed([27819D609D1F050:1F54D9A46892D75B]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNotNull(Assert.java:526) at org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup(ScheduledMaintenanceTriggerTest.java:253) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test Error Message: Timeout occured while waiting response from server at: http://127.0.0.1:38640/x_okt/i/collection1 Stack Trace: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:38640/x_okt/i/collection1 at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1590) at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:215) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542) ... 53 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest Error Message: 4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest: 1) Thread[id=17993, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 3) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=17991, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest: 1) Thread[id=17993, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 3) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=17991, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 2) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 2) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0) FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test Error Message: There are still nodes recoverying - waited for 320 seconds Stack Trace: java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477) at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit Error Message: Java heap space Stack Trace: java.lang.OutOfMemoryError: Java heap space at __randomizedtesting.SeedInfo.seed([27819D609D1F050:31CA311204662AE7]:0) at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84) at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57) at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168) at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154) at org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278) at org.apache.lucene.store.MockIndexOutputWrapper.copyBytes(MockIndexOutputWrapper.java:165) at org.apache.lucene.codecs.simpletext.SimpleTextCompoundFormat.write(SimpleTextCompoundFormat.java:186) at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4997) at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4510) at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068) at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40) at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2170) at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5103) at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1612) at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1228) at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:189) at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:66) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) FAILED: org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned Error Message: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> Stack Trace: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html> at __randomizedtesting.SeedInfo.seed([9919E1BF1324B5F4:61DF188A803D2D3C]:0) at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237) at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned(CloudSolrClientTest.java:725) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985) at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apa [...truncated too long message...] [junit4] 1> </body> [junit4] 1> </html> [junit4] 1> , retry? 3 [junit4] 1> 22:26:38.424 [TEST-CloudSolrClientTest.testVersionsAreReturned-seed#[9919E1BF1324B5F4]] ERROR org.apache.solr.client.solrj.impl.CloudSolrClient - Request to collection [collection1] failed due to (404) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> [junit4] 1> <head> [junit4] 1> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> [junit4] 1> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> [junit4] 1> </head> [junit4] 1> <body><h2>HTTP ERROR 404</h2> [junit4] 1> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: [junit4] 1> <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> [junit4] 1> [junit4] 1> </body> [junit4] 1> </html> [junit4] 1> , retry? 4 [junit4] 1> 22:26:38.430 [TEST-CloudSolrClientTest.testVersionsAreReturned-seed#[9919E1BF1324B5F4]] ERROR org.apache.solr.client.solrj.impl.CloudSolrClient - Request to collection [collection1] failed due to (404) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> [junit4] 1> <head> [junit4] 1> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> [junit4] 1> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> [junit4] 1> </head> [junit4] 1> <body><h2>HTTP ERROR 404</h2> [junit4] 1> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: [junit4] 1> <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> [junit4] 1> [junit4] 1> </body> [junit4] 1> </html> [junit4] 1> , retry? 5 [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory. [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.method=testVersionsAreReturned -Dtests.seed=9919E1BF1324B5F4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=de-DE -Dtests.timezone=America/Indiana/Petersburg -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] ERROR 4.66s J2 | CloudSolrClientTest.testVersionsAreReturned <<< [junit4] > Throwable #1: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> [junit4] > <head> [junit4] > <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> [junit4] > <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> [junit4] > </head> [junit4] > <body><h2>HTTP ERROR 404</h2> [junit4] > <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: [junit4] > <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> [junit4] > </body> [junit4] > </html> [junit4] > at __randomizedtesting.SeedInfo.seed([9919E1BF1324B5F4:61DF188A803D2D3C]:0) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817) [junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) [junit4] > at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned(CloudSolrClientTest.java:725) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> [junit4] > <head> [junit4] > <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> [junit4] > <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> [junit4] > </head> [junit4] > <body><h2>HTTP ERROR 404</h2> [junit4] > <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: [junit4] > <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty";>Powered by Jetty:// 9.4.11.v20180605</a><hr/> [junit4] > </body> [junit4] > </html> [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607) [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528) [junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] > ... 1 more [junit4] 1> 22:26:51.151 [SUITE-CloudSolrClientTest-seed#[9919E1BF1324B5F4]-worker] ERROR org.apache.zookeeper.server.ZooKeeperServer - ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.impl.CloudSolrClientTest_9919E1BF1324B5F4-001 [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), title_s=Lucene50(blocksize=128), id=FST50, text=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Lucene70), a_t=DocValuesFormat(name=Lucene70), title_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Lucene70), text=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1316, maxMBSortInHeap=5.869300803513228, sim=RandomSimilarity(queryNorm=false): {}, locale=de-DE, timezone=America/Indiana/Petersburg [junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=179963528,total=438829056 [junit4] 2> NOTE: All tests run in this JVM: [HttpSolrClientBadInputTest, SolrExampleStreamingTest, ConcurrentUpdateSolrClientMultiCollectionTest, QueryResponseTest, LessThanEvaluatorTest, NotEvaluatorTest, TestLang, JdbcDriverTest, TestSolrJErrorHandling, ClientUtilsTest, AscEvaluatorTest, EmpiricalDistributionEvaluatorTest, StreamingTest, TestDelegationTokenResponse, ArcSineEvaluatorTest, SolrExampleXMLTest, FieldValueEvaluatorTest, AndEvaluatorTest, TestPathTrie, SquareRootEvaluatorTest, LargeVolumeEmbeddedTest, CollectionAdminRequestRequiredParamsTest, StreamExpressionParserTest, MathExpressionTest, StreamExpressionTest, FacetFieldTest, CloudSolrClientTest] [junit4] Completed [140/162 (1!)] on J2 in 72.71s, 16 tests, 1 error <<< FAILURES! [...truncated 7950 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:661: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created: * java_pid22393.hprof Total time: 281 minutes 30 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
