Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1053/
12 tests failed. FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test Error Message: IOException occured when talking to server at: https://127.0.0.1:38427/h_ Stack Trace: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:38427/h_ at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:620) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:417) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:38427 [/127.0.0.1] failed: Read timed out at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143) at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353) at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:511) ... 47 more Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) at sun.security.ssl.InputRecord.read(InputRecord.java:503) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394) at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353) at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134) ... 57 more FAILED: org.apache.solr.handler.TestReplicationHandler.doTestStressReplication Error Message: timed out waiting for collection1 startAt time to exceed: Mon Jun 27 00:04:55 NFT 2016 Stack Trace: java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Mon Jun 27 00:04:55 NFT 2016 at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96:A166C77D08AC4225]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1506) at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:856) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: org.apache.solr.util.TestSolrCLIRunExample.testInteractiveSolrCloudExample Error Message: Expected 10 to be found in the testCloudExamplePrompt collection but only found 8 Stack Trace: java.lang.AssertionError: Expected 10 to be found in the testCloudExamplePrompt collection but only found 8 at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96:A1BC27713AF1EEF0]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.util.TestSolrCLIRunExample.testInteractiveSolrCloudExample(TestSolrCLIRunExample.java:457) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationHandlerTest Error Message: ObjectTracker found 12 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] Stack Trace: java.lang.AssertionError: ObjectTracker found 12 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:256) at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationHandlerTest Error Message: 12 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrReplicationHandlerTest: 1) Thread[id=4115, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=3608, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=2949, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=3849, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=3609, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=3221, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=3848, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=3486, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=3487, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=3222, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=4114, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=2948, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrReplicationHandlerTest: 1) Thread[id=4115, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=3608, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=2949, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=3849, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=3609, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=3221, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=3848, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=3486, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=3487, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=3222, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=4114, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=2948, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest Error Message: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] Stack Trace: java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:256) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest Error Message: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 1) Thread[id=4464, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=4439, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=4429, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=4434, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=4419, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=4444, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=4424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=4449, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=4454, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=4459, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 1) Thread[id=4464, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=4439, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=4429, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=4434, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=4419, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=4444, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=4424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=4449, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=4454, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=4459, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test Error Message: KeeperErrorCode = Session expired for /clusterstate.json Stack Trace: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:347) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:344) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:344) at org.apache.solr.common.cloud.ZkStateReader.refreshLegacyClusterState(ZkStateReader.java:506) at org.apache.solr.common.cloud.ZkStateReader.forceUpdateCollection(ZkStateReader.java:296) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoCollectionSpecified(CollectionsAPIDistributedZkTest.java:583) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:181) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest Error Message: ObjectTracker found 48 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] Stack Trace: java.lang.AssertionError: ObjectTracker found 48 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:256) at sun.reflect.GeneratedMethodAccessor55.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest Error Message: 48 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrReplicationDistributedZkTest: 1) Thread[id=31608, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=31139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=31428, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=35397, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=31609, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=38774, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=38024, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=31853, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=38917, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=36519, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=38915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=31426, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=39159, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=39161, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=38025, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=31425, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=38490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=34941, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=34944, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=31852, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=31851, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=38488, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=38916, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=31610, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=31141, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=31850, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=31142, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=35398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=31427, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=36516, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=36517, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=34943, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=38918, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 34) Thread[id=36518, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=38026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=35399, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=39158, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=39160, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=38027, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=38773, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=31140, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=38489, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=38776, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=35396, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=31607, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=38491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=34942, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=38775, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 48 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrReplicationDistributedZkTest: 1) Thread[id=31608, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=31139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=31428, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=35397, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=31609, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=38774, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=38024, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=31853, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=38917, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=36519, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=38915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=31426, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=39159, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=39161, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=38025, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=31425, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=38490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=34941, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=34944, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=31852, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=31851, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=38488, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=38916, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=31610, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=31141, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=31850, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=31142, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=35398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=31427, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=36516, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=36517, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=34943, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=38918, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 34) Thread[id=36518, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=38026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=35399, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=39158, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=39160, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=38027, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=38773, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=31140, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=38489, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=38776, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=35396, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=31607, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=38491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=34942, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=38775, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrReplicationDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrRequestHandlerTest Error Message: ObjectTracker found 8 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] Stack Trace: java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:256) at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrRequestHandlerTest Error Message: 8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrRequestHandlerTest: 1) Thread[id=120088, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=120473, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=120089, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=120087, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=120090, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=120472, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=120471, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=120474, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrRequestHandlerTest: 1) Thread[id=120088, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=120473, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=120089, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=120087, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=120090, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=120472, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=120471, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=120474, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrRequestHandlerTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96]:0) Build Log: [...truncated 11246 lines...] [junit4] Suite: org.apache.solr.cloud.CdcrReplicationHandlerTest [junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationHandlerTest_7ACDC7BB0D842B96-001/init-core-data-001 [junit4] 2> 295675 INFO (SUITE-CdcrReplicationHandlerTest-seed#[7ACDC7BB0D842B96]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 295676 INFO (SUITE-CdcrReplicationHandlerTest-seed#[7ACDC7BB0D842B96]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /nu_v/ [junit4] 2> 295677 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 295689 INFO (Thread-1631) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 295689 INFO (Thread-1631) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 295789 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.ZkTestServer start zk server on port:41618 [junit4] 2> 295789 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 295789 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 295792 INFO (zkCallback-241-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@f0aca83 name:ZooKeeperConnection Watcher:127.0.0.1:41618 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 295792 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 295792 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 295792 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 295794 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 295794 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 295795 INFO (zkCallback-242-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@30ce0b1d name:ZooKeeperConnection Watcher:127.0.0.1:41618/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 295795 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 295795 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 295796 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1 [junit4] 2> 295797 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards [junit4] 2> 295798 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection [junit4] 2> 295798 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards [junit4] 2> 295799 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml [junit4] 2> 295799 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml [junit4] 2> 295800 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationWithTruncatedTlog-seed#[7A [...truncated too long message...] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:330) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:327) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:327) [junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:105) [junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) [junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:265) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 6490582 INFO (jetty-launcher-30203-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@32d163c4{/solr,null,UNAVAILABLE} [junit4] 2> 6490585 INFO (TEST-TestSolrCLIRunExample.testInteractiveSolrCloudExample-seed#[7ACDC7BB0D842B96]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59327 59327 [junit4] 2> 6490750 INFO (Thread-15258) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59327 59327 [junit4] 2> 6490751 WARN (Thread-15258) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 9 /solr/configs/testCloudExamplePrompt/managed-schema [junit4] 2> 4 /solr/aliases.json [junit4] 2> 4 /solr/clusterprops.json [junit4] 2> 2 /solr/configs/testCloudExamplePrompt [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 4 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 13 /solr/overseer/queue [junit4] 2> 11 /solr/overseer/collection-queue-work [junit4] 2> 4 /solr/live_nodes [junit4] 2> 4 /solr/collections [junit4] 2> 2 /solr/overseer/queue-work [junit4] 2> [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory. [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCLIRunExample -Dtests.method=testInteractiveSolrCloudExample -Dtests.seed=7ACDC7BB0D842B96 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/x1/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar-KW -Dtests.timezone=Asia/Yekaterinburg -Dtests.asserts=true -Dtests.file.encoding=US-ASCII [junit4] FAILURE 13.3s J2 | TestSolrCLIRunExample.testInteractiveSolrCloudExample <<< [junit4] > Throwable #1: java.lang.AssertionError: Expected 10 to be found in the testCloudExamplePrompt collection but only found 8 [junit4] > at __randomizedtesting.SeedInfo.seed([7ACDC7BB0D842B96:A1BC27713AF1EEF0]:0) [junit4] > at org.apache.solr.util.TestSolrCLIRunExample.testInteractiveSolrCloudExample(TestSolrCLIRunExample.java:457) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] 2> 6490754 INFO (SUITE-TestSolrCLIRunExample-seed#[7ACDC7BB0D842B96]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/build/solr-core/test/J2/temp/solr.util.TestSolrCLIRunExample_7ACDC7BB0D842B96-001 [junit4] 2> Jun 26, 2016 2:00:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=200, maxMBSortInHeap=6.666143493389538, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=ar-KW, timezone=Asia/Yekaterinburg [junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_74 (64-bit)/cpus=4,threads=1,free=74268040,total=505937920 [junit4] 2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, TestManagedResource, TestIndexSearcher, SortSpecParsingTest, TestSolrDynamicMBean, TestSolrJ, QueryParsingTest, TestMacros, HdfsChaosMonkeyNothingIsSafeTest, TestSolrQueryParser, ResponseHeaderTest, DirectUpdateHandlerOptimizeTest, TestReversedWildcardFilterFactory, AssignTest, TestExactSharedStatsCache, NoCacheHeaderTest, TermVectorComponentDistributedTest, PreAnalyzedUpdateProcessorTest, RollingRestartTest, BooleanFieldTest, TestScoreJoinQPNoScore, TestDefaultSearchFieldResource, TestExtendedDismaxParser, SuggesterTSTTest, TestHighFrequencyDictionaryFactory, ExplicitHLLTest, BaseCdcrDistributedZkTest, LukeRequestHandlerTest, TestSuggestSpellingConverter, UUIDUpdateProcessorFallbackTest, BasicAuthIntegrationTest, DistributedFacetPivotLongTailTest, CollectionsAPISolrJTest, DistributedSuggestComponentTest, TestCloudSchemaless, LeaderElectionTest, SampleTest, ZkStateWriterTest, TestFieldCacheVsDocValues, TestSQLHandler, TestTolerantUpdateProcessorRandomCloud, TestDistributedMissingSort, TestTolerantSearch, TestRandomRequestDistribution, TestWriterPerf, TestPostingsSolrHighlighter, FieldMutatingUpdateProcessorTest, ScriptEngineTest, RecoveryZkTest, TestLockTree, DirectUpdateHandlerTest, BasicDistributedZk2Test, ShardSplitTest, TestZkChroot, DocValuesMultiTest, TestRecovery, PrimitiveFieldTypeTest, MigrateRouteKeyTest, BlockJoinFacetRandomTest, DataDrivenBlockJoinTest, TestCSVResponseWriter, WrapperMergePolicyFactoryTest, AddBlockUpdateTest, CursorMarkTest, TestDocTermOrds, RequestHandlersTest, TestCollapseQParserPlugin, TestGroupingSearch, VMParamsZkACLAndCredentialsProvidersTest, SmileWriterTest, TestStressUserVersions, PluginInfoTest, CleanupOldIndexTest, TestCloudInspectUtil, TestDFISimilarityFactory, JavabinLoaderTest, SuggesterTest, TestMissingGroups, TestManagedSchemaAPI, TestManagedStopFilterFactory, TestSubQueryTransformer, OverriddenZkACLAndCredentialsProvidersTest, RulesTest, TestExclusionRuleCollectionAccess, TestJmxIntegration, ConnectionManagerTest, TestReload, TestConfigSetsAPI, TestStressCloudBlindAtomicUpdates, TestSortByMinMaxFunction, StatelessScriptUpdateProcessorFactoryTest, FullSolrCloudDistribCmdsTest, StatsComponentTest, TestFastWriter, TestHdfsUpdateLog, BadComponentTest, CoreAdminHandlerTest, XmlUpdateRequestHandlerTest, RegexBoostProcessorTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, RestartWhileUpdatingTest, TestRequestForwarding, TestShortCircuitedRequests, TestStressLiveNodes, HdfsChaosMonkeySafeLeaderTest, TestClusterStateMutator, ZkStateReaderTest, RuleEngineTest, BlobRepositoryCloudTest, CachingDirectoryFactoryTest, ExitableDirectoryReaderTest, HdfsDirectoryFactoryTest, TestFieldCacheSanityChecker, TestFieldCacheSort, TestFieldCacheSortRandom, TestNumericTerms32, TestUninvertingReader, TestDocBasedVersionConstraints, AddSchemaFieldsUpdateProcessorFactoryTest, AtomicUpdatesTest, ClassificationUpdateProcessorFactoryTest, CloneFieldUpdateProcessorFactoryTest, DocExpirationUpdateProcessorFactoryTest, ParsingFieldUpdateProcessorsTest, TestNamedUpdateProcessors, TestRTimerTree, TestSolrCLIRunExample] [junit4] Completed [609/616 (8!)] on J2 in 16.04s, 3 tests, 1 failure, 1 skipped <<< FAILURES! [...truncated 42 lines...] [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/build/solr-core/test/temp/junit4-J0-20160626_121239_600.sysout [junit4] >>> JVM J0 emitted unexpected output (verbatim) ---- [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/heapdumps/java_pid3383.hprof ... [junit4] Heap dump file created [603400833 bytes in 4.651 secs] [junit4] <<< JVM J0: EOF ---- [...truncated 17 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/build.xml:747: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/build.xml:684: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/build.xml:59: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/build.xml:233: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/solr/common-build.xml:530: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/lucene/common-build.xml:1427: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/lucene/common-build.xml:984: There were test failures: 616 suites (10 ignored), 2590 tests, 8 suite-level errors, 2 errors, 2 failures, 33 ignored (18 assumptions) [seed: 7ACDC7BB0D842B96] Total time: 295 minutes 36 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts No prior successful build to compare, so performing full copy of artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org
