Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/165/
8 tests failed. FAILED: org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation Error Message: Timeout waiting for CDCR replication to complete @source_collection:shard2 Stack Trace: java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2 at __randomizedtesting.SeedInfo.seed([D9DB264E81824972:27B47EED43A26A63]:0) at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794) at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:377) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete Error Message: Timeout waiting for CDCR replication to complete @source_collection:shard1 Stack Trace: java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1 at __randomizedtesting.SeedInfo.seed([D9DB264E81824972:AEC15FEA299B485E]:0) at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794) at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:525) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test Error Message: Timeout occured while waiting response from server at: http://127.0.0.1:35976 Stack Trace: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35976 at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:603) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:515) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160) at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84) at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261) at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251) at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124) at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:497) ... 47 more FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test Error Message: Captured an uncaught exception in thread: Thread[id=121168, name=collection0, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest] Stack Trace: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=121168, name=collection0, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest] Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:46021 at __randomizedtesting.SeedInfo.seed([D9DB264E81824972]:0) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:997) Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:46021 at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:603) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:405) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:357) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1289) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1059) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1001) at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599) at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620) at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:987) Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) at sun.security.ssl.InputRecord.read(InputRecord.java:503) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930) at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160) at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84) at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261) at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251) at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124) at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:497) ... 11 more FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest Error Message: ObjectTracker found 32 object(s) that were not released!!! [NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory] Stack Trace: java.lang.AssertionError: ObjectTracker found 32 object(s) that were not released!!! [NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, MDCAwareThreadPoolExecutor, NRTCachingDirectory, MDCAwareThreadPoolExecutor, MDCAwareThreadPoolExecutor, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory] at __randomizedtesting.SeedInfo.seed([D9DB264E81824972]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertNull(Assert.java:551) at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest Error Message: 8 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest: 1) Thread[id=124923, name=searcherExecutor-9969-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=124922, name=searcherExecutor-9934-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=124809, name=searcherExecutor-9983-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=124875, name=searcherExecutor-9955-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=124813, name=searcherExecutor-9963-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=124925, name=searcherExecutor-9950-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=124862, name=searcherExecutor-9986-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=124811, name=searcherExecutor-9982-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest: 1) Thread[id=124923, name=searcherExecutor-9969-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=124922, name=searcherExecutor-9934-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=124809, name=searcherExecutor-9983-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=124875, name=searcherExecutor-9955-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=124813, name=searcherExecutor-9963-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=124925, name=searcherExecutor-9950-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=124862, name=searcherExecutor-9986-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=124811, name=searcherExecutor-9982-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([D9DB264E81824972]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=124923, name=searcherExecutor-9969-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=124922, name=searcherExecutor-9934-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=124809, name=searcherExecutor-9983-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=124875, name=searcherExecutor-9955-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=124813, name=searcherExecutor-9963-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=124925, name=searcherExecutor-9950-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=124862, name=searcherExecutor-9986-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=124811, name=searcherExecutor-9982-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=124923, name=searcherExecutor-9969-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=124922, name=searcherExecutor-9934-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=124809, name=searcherExecutor-9983-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=124875, name=searcherExecutor-9955-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=124813, name=searcherExecutor-9963-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=124925, name=searcherExecutor-9950-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=124862, name=searcherExecutor-9986-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=124811, name=searcherExecutor-9982-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at __randomizedtesting.SeedInfo.seed([D9DB264E81824972]:0) FAILED: org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart Error Message: Timeout waiting for CDCR replication to complete @source_collection:shard1 Stack Trace: java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1 at __randomizedtesting.SeedInfo.seed([D9DB264E81824972:85C6C1CF4FA2FD44]:0) at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794) at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:235) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) Build Log: [...truncated 11604 lines...] [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest [junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/init-core-data-001 [junit4] 2> 787775 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[D9DB264E81824972]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 787775 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[D9DB264E81824972]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: / [junit4] 2> 787794 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 787834 INFO (Thread-1157) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 787834 INFO (Thread-1157) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 787920 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.ZkTestServer start zk server on port:60638 [junit4] 2> 787920 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 787996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788044 INFO (zkCallback-471-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@602e1ccd name:ZooKeeperConnection Watcher:127.0.0.1:60638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788044 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788044 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788044 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 788052 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 788057 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788057 INFO (zkCallback-472-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@36cd86b5 name:ZooKeeperConnection Watcher:127.0.0.1:60638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788057 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788058 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788058 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1 [junit4] 2> 788059 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards [junit4] 2> 788080 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection [junit4] 2> 788081 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards [junit4] 2> 788083 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml [junit4] 2> 788083 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml [junit4] 2> 788093 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml [junit4] 2> 788094 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml [junit4] 2> 788095 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 788095 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 788097 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt [junit4] 2> 788097 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt [junit4] 2> 788098 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt [junit4] 2> 788098 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt [junit4] 2> 788099 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml [junit4] 2> 788099 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml [junit4] 2> 788100 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml [junit4] 2> 788101 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml [junit4] 2> 788109 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json [junit4] 2> 788109 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json [junit4] 2> 788110 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 788111 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 788112 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt [junit4] 2> 788112 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt [junit4] 2> 788116 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt [junit4] 2> 788116 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt [junit4] 2> 788133 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 788135 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788135 INFO (zkCallback-473-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@14dd7210 name:ZooKeeperConnection Watcher:127.0.0.1:60638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788135 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788135 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788135 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 788137 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 788138 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788139 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@17aad671 name:ZooKeeperConnection Watcher:127.0.0.1:60638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788139 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788139 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788139 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml [junit4] 2> 788139 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml [junit4] 2> 788141 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml [junit4] 2> 788141 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml [junit4] 2> 788142 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml [junit4] 2> 788143 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml [junit4] 2> 788148 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 788149 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 788150 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt [junit4] 2> 788150 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt [junit4] 2> 788151 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt [junit4] 2> 788151 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt [junit4] 2> 788152 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml [junit4] 2> 788152 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml [junit4] 2> 788154 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json [junit4] 2> 788154 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json [junit4] 2> 788155 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt [junit4] 2> 788155 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt [junit4] 2> 788161 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt [junit4] 2> 788161 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt [junit4] 2> 788162 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt [junit4] 2> 788162 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt [junit4] 2> 788168 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml [junit4] 2> 788168 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml [junit4] 2> 788572 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/cores/collection1 [junit4] 2> 788577 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.e.j.s.Server jetty-9.3.8.v20160314 [junit4] 2> 788579 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16e4bc17{/,null,AVAILABLE} [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.e.j.s.ServerConnector Started ServerConnector@111b4880{HTTP/1.1,[http/1.1]}{127.0.0.1:56355} [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.e.j.s.Server Started @796452ms [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/tempDir-001/control/data, hostContext=/, hostPort=56355, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/cores} [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93 [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001' [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx) [junit4] 2> 788585 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI) [junit4] 2> 788586 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 788608 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788609 INFO (zkCallback-475-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3a9f7d91 name:ZooKeeperConnection Watcher:127.0.0.1:60638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788629 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper) [junit4] 2> 788629 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/solr.xml [junit4] 2> 788660 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/cores [junit4] 2> 788660 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.CoreContainer New CoreContainer 1499133815 [junit4] 2> 788660 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001] [junit4] 2> 788660 WARN (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/lib [junit4] 2> 788660 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,connectionsEvictorSleepDelay : 5000,maxConnectionIdleTime : 40000, [junit4] 2> 788664 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true [junit4] 2> 788665 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 788665 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 788665 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60638/solr [junit4] 2> 788665 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.ZkController zkHost includes chroot [junit4] 2> 788665 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 788666 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788667 INFO (zkCallback-478-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@265c6af0 name:ZooKeeperConnection Watcher:127.0.0.1:60638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788667 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788667 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 788691 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$63/1132055862@41e6ddf9 [junit4] 2> 788692 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 788696 INFO (zkCallback-479-thread-1-processing-n:127.0.0.1:56355_) [n:127.0.0.1:56355_ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@28cb4360 name:ZooKeeperConnection Watcher:127.0.0.1:60638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 788696 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 788720 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 788763 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 788828 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 788830 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 788831 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 788832 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 788833 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 788834 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json [junit4] 2> 788841 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /security.json [junit4] 2> 788842 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 788843 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties [junit4] 2> 788843 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 788845 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect [junit4] 2> 788847 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election [junit4] 2> 788848 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 788849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96625693769072646-127.0.0.1:56355_-n_0000000000 [junit4] 2> 788849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56355_ [junit4] 2> 788849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader [junit4] 2> 788850 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.Overseer Overseer (id=96625693769072646-127.0.0.1:56355_-n_0000000000) starting [junit4] 2> 788852 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work [junit4] 2> 788873 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000 [junit4] 2> 788964 INFO (OverseerCollectionConfigSetProcessor-96625693769072646-127.0.0.1:56355_-n_0000000000) [n:127.0.0.1:56355_ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations [junit4] 2> 788965 INFO (OverseerStateUpdate-96625693769072646-127.0.0.1:56355_-n_0000000000) [n:127.0.0.1:56355_ ] o.a.s.c.Overseer Starting to work on the main queue [junit4] 2> 788975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56355_ [junit4] 2> 788975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:56355_ [junit4] 2> 788976 INFO (zkCallback-479-thread-1-processing-n:127.0.0.1:56355_) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 788977 INFO (zkCallback-479-thread-1-processing-n:127.0.0.1:56355_) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 788988 INFO (OverseerStateUpdate-96625693769072646-127.0.0.1:56355_-n_0000000000) [n:127.0.0.1:56355_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1) [junit4] 2> 788989 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module. [junit4] 2> 788989 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CoreContainer No authentication plugin used. [junit4] 2> 788990 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/cores [junit4] 2> 788991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=} [junit4] 2> 788991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_D9DB264E81824972-001/control-001/cores/collection1 [junit4] 2> 788991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions [junit4] 2> 789065 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1 [junit4] 2> 789065 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[D9DB264E81824972]) [n:127.0.0.1:56355_ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done [junit4] 2> 789065 INFO (coreLoadExecutor-1430-thread-1-processing-n:127.0.0.1:56355_) [n:127.0.0.1:56355_ c:control_collection x:collection1] o.a.s.c.ZkController publishing state=down [...truncated too long message...] :57847_ c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_D9DB264E81824972-001/jetty-088/cores/target_collection_shard2_replica1/data [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_D9DB264E81824972-001/jetty-088/cores/target_collection_shard2_replica1/data;done=false>>] [junit4] 2> 8075645 INFO (coreCloseExecutor-12409-thread-1) [n:127.0.0.1:57847_ c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true [junit4] 2> 8075645 INFO (coreCloseExecutor-12409-thread-1) [n:127.0.0.1:57847_ c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_D9DB264E81824972-001/jetty-088/cores/target_collection_shard2_replica1/data [junit4] 2> 8075647 INFO (TEST-CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange-seed#[D9DB264E81824972]) [ ] o.a.s.c.Overseer Overseer (id=96626155173117985-127.0.0.1:57847_-n_0000000007) closing [junit4] 2> 8075647 INFO (OverseerStateUpdate-96626155173117985-127.0.0.1:57847_-n_0000000007) [n:127.0.0.1:57847_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57847_ [junit4] 2> 8075648 INFO (zkCallback-30506-thread-4-processing-n:127.0.0.1:57847_) [n:127.0.0.1:57847_ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 8075648 WARN (zkCallback-30506-thread-4-processing-n:127.0.0.1:57847_) [n:127.0.0.1:57847_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 8075653 INFO (TEST-CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange-seed#[D9DB264E81824972]) [ ] o.e.j.s.ServerConnector Stopped ServerConnector@f759056{SSL,[ssl, http/1.1]}{127.0.0.1:0} [junit4] 2> 8075654 INFO (TEST-CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange-seed#[D9DB264E81824972]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4f71668b{/,null,UNAVAILABLE} [junit4] 2> 8075654 INFO (TEST-CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange-seed#[D9DB264E81824972]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testReplicationAfterLeaderChange [junit4] 2> 8075655 INFO (TEST-CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange-seed#[D9DB264E81824972]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46240 46240 [junit4] 2> 8075894 INFO (Thread-7146) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46240 46240 [junit4] 2> 8075894 WARN (Thread-7146) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 42 /solr/aliases.json [junit4] 2> 20 /solr/configs/conf1 [junit4] 2> 12 /solr/security.json [junit4] 2> 8 /solr/collections/tmp_collection/state.json [junit4] 2> 4 /solr/collections/target_collection/leaders/shard2/leader [junit4] 2> 3 /solr/collections/source_collection/leaders/shard2/leader [junit4] 2> 3 /solr/collections/source_collection/leaders/shard1/leader [junit4] 2> 3 /solr/collections/target_collection/leaders/shard1/leader [junit4] 2> 2 /solr/collections/tmp_collection/leaders/shard2/leader [junit4] 2> 2 /solr/collections/tmp_collection/leaders/shard1/leader [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 42 /solr/clusterprops.json [junit4] 2> 42 /solr/clusterstate.json [junit4] 2> 9 /solr/collections/target_collection/cdcr/state/buffer [junit4] 2> 9 /solr/collections/source_collection/cdcr/state/process [junit4] 2> 9 /solr/collections/target_collection/cdcr/state/process [junit4] 2> 8 /solr/collections/tmp_collection/cdcr/state/process [junit4] 2> 8 /solr/collections/source_collection/cdcr/state/buffer [junit4] 2> 8 /solr/collections/tmp_collection/cdcr/state/buffer [junit4] 2> 6 /solr/collections/source_collection/state.json [junit4] 2> 6 /solr/collections/target_collection/state.json [junit4] 2> 2 /solr/overseer_elect/election/96626155173117966-127.0.0.1:38578_-n_0000000003 [junit4] 2> 2 /solr/overseer_elect/election/96626155173117985-127.0.0.1:57847_-n_0000000007 [junit4] 2> 2 /solr/overseer_elect/election/96626155173117960-127.0.0.1:40059_-n_0000000001 [junit4] 2> 2 /solr/overseer_elect/election/96626155173117979-127.0.0.1:39507_-n_0000000005 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 225 /solr/overseer/collection-queue-work [junit4] 2> 94 /solr/overseer/queue [junit4] 2> 42 /solr/collections [junit4] 2> 38 /solr/live_nodes [junit4] 2> 36 /solr/overseer/queue-work [junit4] 2> [junit4] 2> 8075896 INFO (SUITE-CdcrReplicationDistributedZkTest-seed#[D9DB264E81824972]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_D9DB264E81824972-001 [junit4] 2> Sep 20, 2016 7:06:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2> NOTE: test params are: codec=CheapBastard, sim=ClassicSimilarity, locale=ar, timezone=America/Nassau [junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=222210040,total=523239424 [junit4] 2> NOTE: All tests run in this JVM: [TestPHPSerializedResponseWriter, NotRequiredUniqueKeyTest, TestBinaryResponseWriter, TestIBSimilarityFactory, ReturnFieldsTest, TestFieldCacheWithThreads, TestDocSet, TestCollapseQParserPlugin, BlockJoinFacetDistribTest, ExternalFileFieldSortTest, TestPivotHelperCode, DocValuesMissingTest, LeaderInitiatedRecoveryOnShardRestartTest, CoreSorterTest, RemoteQueryErrorTest, AsyncCallRequestStatusResponseTest, TestFunctionQuery, TestStressLucene, UUIDFieldTest, TestSolr4Spatial2, ConnectionReuseTest, TestNumericTerms64, TestJsonRequest, CoreMergeIndexesAdminHandlerTest, TestRandomDVFaceting, CdcrRequestHandlerTest, ActionThrottleTest, TestStressRecovery, TestDistribDocBasedVersion, TestPerFieldSimilarityClassic, TestFieldTypeResource, TestClusterStateMutator, OverseerModifyCollectionTest, VersionInfoTest, SharedFSAutoReplicaFailoverUtilsTest, TestStressLiveNodes, AlternateDirectoryTest, TestSolrConfigHandlerConcurrent, TestHdfsBackupRestoreCore, DisMaxRequestHandlerTest, TestManagedSchemaAPI, TestSchemaSimilarityResource, OutOfBoxZkACLAndCredentialsProvidersTest, BlockCacheTest, TestExactStatsCache, TestRangeQuery, DistributedFacetExistsSmallTest, HighlighterConfigTest, TestConfigOverlay, TestSolrCoreProperties, TestCloudDeleteByQuery, TestConfigSetsAPIExclusivity, SolrCoreTest, TestInfoStreamLogging, TestHashQParserPlugin, SparseHLLTest, ChaosMonkeyNothingIsSafeTest, PolyFieldTest, CurrencyFieldOpenExchangeTest, SortSpecParsingTest, TestRestoreCore, NumericFieldsTest, CacheHeaderTest, TestCodecSupport, TestDocBasedVersionConstraints, LeaderElectionIntegrationTest, FastVectorHighlighterTest, TestAtomicUpdateErrorCases, DateFieldTest, DistributedFacetPivotSmallAdvancedTest, TestFaceting, HighlighterMaxOffsetTest, TlogReplayBufferedWhileIndexingTest, TestSolrQueryResponse, TestBulkSchemaAPI, TestOrdValues, TestLocalFSCloudBackupRestore, TestRuleBasedAuthorizationPlugin, TestFieldCollectionResource, DefaultValueUpdateProcessorTest, TestSolr4Spatial, AnalysisAfterCoreReloadTest, BasicDistributedZkTest, SaslZkACLProviderTest, CleanupOldIndexTest, QueryElevationComponentTest, TestConfigSets, CursorPagingTest, HdfsUnloadDistributedZkTest, TestSearcherReuse, CollectionStateFormat2Test, HdfsDirectoryFactoryTest, TestRandomFaceting, TestQuerySenderNoQuery, HdfsRecoverLeaseTest, RegexBoostProcessorTest, TestSolrQueryParserDefaultOperatorResource, TestReplicationHandler, IgnoreCommitOptimizeUpdateProcessorFactoryTest, HttpPartitionTest, OverseerTaskQueueTest, TestPseudoReturnFields, TestMacros, ClusterStateTest, TestDistributedGrouping, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestJoin, StatsComponentTest, TestFiltering, SoftAutoCommitTest, CurrencyFieldXmlFileTest, TestCoreDiscovery, SpellCheckCollatorTest, SolrRequestParserTest, SuggesterWFSTTest, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestUpdate, TestWordDelimiterFilterFactory, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, StandardRequestHandlerTest, TestWriterPerf, TestQueryTypes, DocumentBuilderTest, TestIndexingPerformance, TestSolrQueryParser, SolrPluginUtilsTest, TestJmxIntegration, MBeansHandlerTest, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestComponentsName, TestLFUCache, CdcrReplicationDistributedZkTest] [junit4] Completed [630/630 (3!)] on J0 in 1917.37s, 11 tests, 3 errors <<< FAILURES! [...truncated 60256 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org