Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1279/

4 tests failed.
FAILED:  
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
expected:<3> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<1>
        at 
__randomizedtesting.SeedInfo.seed([3BBD3F23147BAA1F:73C84B971248858A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:523)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
.response.numFound:36!=67

Stack Trace:
junit.framework.AssertionFailedError: .response.numFound:36!=67
        at 
__randomizedtesting.SeedInfo.seed([3BBD3F23147BAA1F:B3E900F9BA87C7E7]:0)
        at junit.framework.Assert.fail(Assert.java:50)
        at 
org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
        at 
org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:932)
        at 
org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:607)
        at 
org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
        at 
org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkQueries(AbstractFullDistribZkTestBase.java:883)
        at 
org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:98)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [SolrCore, 
NRTCachingDirectory, NRTCachingDirectory] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.core.SolrCore  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1022)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)  at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:353)  at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:715)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:931)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)  at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)  at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:931)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)  at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not 
released!!! [SolrCore, NRTCachingDirectory, NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.core.SolrCore
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1022)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)
        at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:353)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:715)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:931)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)
        at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.NRTCachingDirectory
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
        at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
        at 
org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)
        at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:931)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:849)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:964)
        at 
org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:586)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([3BBD3F23147BAA1F]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
        at sun.reflect.GeneratedMethodAccessor112.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
expected:<3> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<3> but was:<2>
        at 
__randomizedtesting.SeedInfo.seed([3BBD3F23147BAA1F:73C84B971248858A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:523)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12553 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/init-core-data-001
   [junit4]   2> 2860798 WARN  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=36 numCloses=36
   [junit4]   2> 2860799 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 2860800 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 2860802 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001
   [junit4]   2> 2860802 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2860802 INFO  (Thread-8288) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2860802 INFO  (Thread-8288) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 2860903 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:42919
   [junit4]   2> 2860928 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2860930 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2860931 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2860933 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2860934 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@704fb775{/solr,null,AVAILABLE}
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@527ed307{SSL,[ssl, 
http/1.1]}{127.0.0.1:38210}
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.e.j.s.Server Started @2867449ms
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=38210}
   [junit4]   2> 2860935 ERROR (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2860935 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-11T04:52:23.370Z
   [junit4]   2> 2861053 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3273665{/solr,null,AVAILABLE}
   [junit4]   2> 2861053 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@2bbeef0d{SSL,[ssl, 
http/1.1]}{127.0.0.1:45966}
   [junit4]   2> 2861053 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.e.j.s.Server Started @2867568ms
   [junit4]   2> 2861053 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=45966}
   [junit4]   2> 2861054 ERROR (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2861054 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 2861054 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2861054 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2861054 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-11T04:52:23.489Z
   [junit4]   2> 2861135 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1b1691b2{/solr,null,AVAILABLE}
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@58e73dfa{SSL,[ssl, 
http/1.1]}{127.0.0.1:37108}
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.e.j.s.Server Started @2867650ms
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=37108}
   [junit4]   2> 2861136 ERROR (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2861136 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-11T04:52:23.571Z
   [junit4]   2> 2861181 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@221ab55f{/solr,null,AVAILABLE}
   [junit4]   2> 2861181 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@472a2e73{SSL,[ssl, 
http/1.1]}{127.0.0.1:51654}
   [junit4]   2> 2861181 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.e.j.s.Server Started @2867696ms
   [junit4]   2> 2861181 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=51654}
   [junit4]   2> 2861181 ERROR (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 2861182 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
7.0.0
   [junit4]   2> 2861182 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2861182 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2861182 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-04-11T04:52:23.617Z
   [junit4]   2> 2861184 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2861231 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2861241 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2861272 INFO  (jetty-launcher-2846-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42919/solr
   [junit4]   2> 2861286 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2861331 INFO  (jetty-launcher-2846-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42919/solr
   [junit4]   2> 2861356 INFO  (jetty-launcher-2846-thread-4) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42919/solr
   [junit4]   2> 2861428 INFO  (jetty-launcher-2846-thread-3) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42919/solr
   [junit4]   2> 2861535 INFO  (jetty-launcher-2846-thread-1) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2861555 INFO  (jetty-launcher-2846-thread-2) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2861562 INFO  (jetty-launcher-2846-thread-1) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:51654_solr
   [junit4]   2> 2861563 INFO  (jetty-launcher-2846-thread-1) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.Overseer Overseer 
(id=97772263390445577-127.0.0.1:51654_solr-n_0000000000) starting
   [junit4]   2> 2861568 INFO  (jetty-launcher-2846-thread-2) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:38210_solr
   [junit4]   2> 2861570 INFO  
(zkCallback-2865-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2861572 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2861581 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x15b5b5b0446000a, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2861589 INFO  (jetty-launcher-2846-thread-4) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 2861591 INFO  (jetty-launcher-2846-thread-4) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2861598 INFO  (jetty-launcher-2846-thread-4) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:45966_solr
   [junit4]   2> 2861599 INFO  
(zkCallback-2865-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2861601 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2861632 INFO  
(zkCallback-2867-thread-1-processing-n:127.0.0.1:45966_solr) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 2861633 INFO  (jetty-launcher-2846-thread-1) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:51654_solr
   [junit4]   2> 2861634 INFO  
(zkCallback-2867-thread-1-processing-n:127.0.0.1:45966_solr) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2861634 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2861698 INFO  
(zkCallback-2865-thread-2-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 2861714 INFO  (jetty-launcher-2846-thread-3) 
[n:127.0.0.1:37108_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (3)
   [junit4]   2> 2861716 INFO  (jetty-launcher-2846-thread-3) 
[n:127.0.0.1:37108_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2861719 INFO  (jetty-launcher-2846-thread-3) 
[n:127.0.0.1:37108_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:37108_solr
   [junit4]   2> 2861720 INFO  
(zkCallback-2867-thread-1-processing-n:127.0.0.1:45966_solr) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2861720 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2861720 INFO  
(zkCallback-2865-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2861721 INFO  
(zkCallback-2871-thread-1-processing-n:127.0.0.1:37108_solr) 
[n:127.0.0.1:37108_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 2861986 INFO  (jetty-launcher-2846-thread-2) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node2/.
   [junit4]   2> 2861988 INFO  (jetty-launcher-2846-thread-4) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node4/.
   [junit4]   2> 2862079 INFO  (jetty-launcher-2846-thread-1) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node1/.
   [junit4]   2> 2862159 INFO  (jetty-launcher-2846-thread-3) 
[n:127.0.0.1:37108_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node3/.
   [junit4]   2> 2865247 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2865251 INFO  
(SUITE-CollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42919/solr ready
   [junit4]   2> 2865363 INFO  
(TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testZeroNumShards
   [junit4]   2> 2865364 INFO  
(TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2865443 INFO  (qtp810562917-18674) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=10&collection.configName=conf&name=acollection&action=CREATE&numShards=0&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2865471 INFO  
(OverseerThreadFactory-4682-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
acollection
   [junit4]   2> 2865471 ERROR 
(OverseerThreadFactory-4682-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: acollection operation: create 
failed:org.apache.solr.common.SolrException: numShards must be > 0
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:124)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:226)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 2865486 INFO  (qtp810562917-18674) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=10&collection.configName=conf&name=acollection&action=CREATE&numShards=0&wt=javabin&version=2}
 status=400 QTime=43
   [junit4]   2> 2865487 INFO  
(TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testZeroNumShards
   [junit4]   2> 2865508 INFO  
(OverseerCollectionConfigSetProcessor-97772263390445577-127.0.0.1:51654_solr-n_0000000000)
 [n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2865552 INFO  
(TEST-CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting deletePartiallyCreatedCollection
   [junit4]   2> 2865553 INFO  
(TEST-CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2865567 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-002&collection.configName=conf&name=halfdeletedcollection_shard1_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&wt=javabin&version=2
   [junit4]   2> 2865568 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 2865570 INFO  
(OverseerStateUpdate-97772263390445577-127.0.0.1:51654_solr-n_0000000000) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.o.ReplicaMutator Assigning new node to 
shard shard=shard2
   [junit4]   2> 2866637 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2866646 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.IndexSchema [halfdeletedcollection_shard1_replica1] Schema name=minimal
   [junit4]   2> 2866659 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2866659 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard1_replica1' 
using configuration from collection halfdeletedcollection, trusted=true
   [junit4]   2> 2866660 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2866660 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrCore [[halfdeletedcollection_shard1_replica1] ] Opening new 
SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node3/halfdeletedcollection_shard1_replica1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-002/]
   [junit4]   2> 2866937 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 2866938 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2866950 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2866951 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2866962 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@18930617[halfdeletedcollection_shard1_replica1] main]
   [junit4]   2> 2866963 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2866964 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2866964 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2866964 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1564356220707405824
   [junit4]   2> 2866965 INFO  
(searcherExecutor-4691-thread-1-processing-n:127.0.0.1:37108_solr 
x:halfdeletedcollection_shard1_replica1 c:halfdeletedcollection) 
[n:127.0.0.1:37108_solr c:halfdeletedcollection   
x:halfdeletedcollection_shard1_replica1] o.a.s.c.SolrCore 
[halfdeletedcollection_shard1_replica1] Registered new searcher 
Searcher@18930617[halfdeletedcollection_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:37108/solr/halfdeletedcollection_shard1_replica1/
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy 
https://127.0.0.1:37108/solr/halfdeletedcollection_shard1_replica1/ has no 
replicas
   [junit4]   2> 2867061 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2867072 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:37108/solr/halfdeletedcollection_shard1_replica1/ shard2
   [junit4]   2> 2867179 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2867186 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr 
c:halfdeletedcollection   x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-002&collection.configName=conf&name=halfdeletedcollection_shard1_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&wt=javabin&version=2}
 status=0 QTime=1618
   [junit4]   2> 2867187 INFO  (qtp810562917-18672) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params 
name=halfdeletedcollection&action=DELETE&wt=javabin&version=2 and 
sendToOCPQueue=true
   [junit4]   2> 2867198 INFO  
(OverseerThreadFactory-4682-thread-2-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing 
Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 2867243 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard2.replica1, tag=null
   [junit4]   2> 2867244 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.c.SolrCore [halfdeletedcollection_shard1_replica1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@1bfb3fe8
   [junit4]   2> 2867255 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard2.replica1, tag=469450728
   [junit4]   2> 2867266 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=halfdeletedcollection_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=22
   [junit4]   2> 2867973 INFO  (qtp810562917-18672) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={name=halfdeletedcollection&action=DELETE&wt=javabin&version=2} status=0 
QTime=785
   [junit4]   2> 2867975 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2867975 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2867986 INFO  
(OverseerCollectionConfigSetProcessor-97772263390445577-127.0.0.1:51654_solr-n_0000000000)
 [n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2867992 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf&name=halfdeletedcollection&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2868016 INFO  
(OverseerThreadFactory-4682-thread-3-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
halfdeletedcollection
   [junit4]   2> 2868502 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard2_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2868503 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 2868503 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard1_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2868503 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr    
] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 
2147483647 transient cores
   [junit4]   2> 2868613 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2868614 INFO  
(zkCallback-2865-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2869607 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2869611 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2869623 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.IndexSchema [halfdeletedcollection_shard1_replica1] Schema name=minimal
   [junit4]   2> 2869625 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.s.IndexSchema [halfdeletedcollection_shard2_replica1] Schema name=minimal
   [junit4]   2> 2869627 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2869627 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard2_replica1' 
using configuration from collection halfdeletedcollection, trusted=true
   [junit4]   2> 2869630 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2869630 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard1_replica1' 
using configuration from collection halfdeletedcollection, trusted=true
   [junit4]   2> 2869631 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2869631 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SolrCore [[halfdeletedcollection_shard1_replica1] ] Opening new 
SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node2/halfdeletedcollection_shard1_replica1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node2/./halfdeletedcollection_shard1_replica1/data/]
   [junit4]   2> 2869634 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2869634 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SolrCore [[halfdeletedcollection_shard2_replica1] ] Opening new 
SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node1/halfdeletedcollection_shard2_replica1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001/tempDir-001/node1/./halfdeletedcollection_shard2_replica1/data/]
   [junit4]   2> 2869914 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 2869914 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2869915 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2869916 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2869924 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.UpdateLog
   [junit4]   2> 2869924 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2869970 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@4e98523e[halfdeletedcollection_shard1_replica1] main]
   [junit4]   2> 2869971 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2869972 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2869972 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2869972 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1564356223861522432
   [junit4]   2> 2869974 INFO  
(searcherExecutor-4697-thread-1-processing-n:127.0.0.1:38210_solr 
x:halfdeletedcollection_shard1_replica1 s:shard1 c:halfdeletedcollection) 
[n:127.0.0.1:38210_solr c:halfdeletedcollection s:shard1  
x:halfdeletedcollection_shard1_replica1] o.a.s.c.SolrCore 
[halfdeletedcollection_shard1_replica1] Registered new searcher 
Searcher@4e98523e[halfdeletedcollection_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2869979 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2869979 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2869980 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:38210/solr/halfdeletedcollection_shard1_replica1/
   [junit4]   2> 2869981 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2869981 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.SyncStrategy 
https://127.0.0.1:38210/solr/halfdeletedcollection_shard1_replica1/ has no 
replicas
   [junit4]   2> 2869981 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2869982 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2869982 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2869983 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@3b06913e[halfdeletedcollection_shard2_replica1] main]
   [junit4]   2> 2869984 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 2869985 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2869985 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2869985 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1564356223875153920
   [junit4]   2> 2869987 INFO  
(searcherExecutor-4696-thread-1-processing-n:127.0.0.1:51654_solr 
x:halfdeletedcollection_shard2_replica1 s:shard2 c:halfdeletedcollection) 
[n:127.0.0.1:51654_solr c:halfdeletedcollection s:shard2  
x:halfdeletedcollection_shard2_replica1] o.a.s.c.SolrCore 
[halfdeletedcollection_shard2_replica1] Registered new searcher 
Searcher@3b06913e[halfdeletedcollection_shard2_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2869989 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:38210/solr/halfdeletedcollection_shard1_replica1/ shard1
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:51654/solr/halfdeletedcollection_shard2_replica1/
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.SyncStrategy 
https://127.0.0.1:51654/solr/halfdeletedcollection_shard2_replica1/ has no 
replicas
   [junit4]   2> 2869995 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.ShardLeaderElectionContext Found all replicas participating in 
election, clear LIR
   [junit4]   2> 2869999 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:51654/solr/halfdeletedcollection_shard2_replica1/ shard2
   [junit4]   2> 2870103 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2870105 INFO  
(zkCallback-2865-thread-1-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2870140 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2870142 INFO  (qtp1678053343-18652) [n:127.0.0.1:38210_solr 
c:halfdeletedcollection s:shard1  x:halfdeletedcollection_shard1_replica1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard1_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard1&wt=javabin&version=2}
 status=0 QTime=1641
   [junit4]   2> 2870150 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2870153 INFO  (qtp882061671-18683) [n:127.0.0.1:51654_solr 
c:halfdeletedcollection s:shard2  x:halfdeletedcollection_shard2_replica1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard2_replica1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard2&wt=javabin&version=2}
 status=0 QTime=1651
   [junit4]   2> 2870161 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 2870266 INFO  
(zkCallback-2862-thread-1-processing-n:127.0.0.1:38210_solr) 
[n:127.0.0.1:38210_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2870267 INFO  
(zkCallback-2865-thread-2-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/halfdeletedcollection/state.json] for collection 
[halfdeletedcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2871162 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf&name=halfdeletedcollection&action=CREATE&numShards=2&wt=javabin&version=2}
 status=0 QTime=3169
   [junit4]   2> 2871162 INFO  (qtp810562917-18678) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2871163 INFO  (qtp810562917-18678) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2871163 INFO  
(TEST-CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending deletePartiallyCreatedCollection
   [junit4]   2> 2871298 INFO  
(TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testTooManyReplicas
   [junit4]   2> 2871299 INFO  
(TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2871300 INFO  (qtp810562917-18674) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params 
name=halfdeletedcollection&action=DELETE&wt=javabin&version=2 and 
sendToOCPQueue=true
   [junit4]   2> 2871312 INFO  
(OverseerCollectionConfigSetProcessor-97772263390445577-127.0.0.1:51654_solr-n_0000000000)
 [n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2871313 INFO  
(OverseerThreadFactory-4682-thread-4-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing 
Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 2871317 INFO  (qtp1678053343-18653) [n:127.0.0.1:38210_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard1.replica1, tag=null
   [junit4]   2> 2871318 INFO  (qtp1678053343-18653) [n:127.0.0.1:38210_solr    
] o.a.s.c.SolrCore [halfdeletedcollection_shard1_replica1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@22720718
   [junit4]   2> 2871319 INFO  (qtp882061671-18684) [n:127.0.0.1:51654_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard2.replica1, tag=null
   [junit4]   2> 2871422 INFO  (qtp882061671-18684) [n:127.0.0.1:51654_solr    
] o.a.s.c.SolrCore [halfdeletedcollection_shard2_replica1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@4915efc4
   [junit4]   2> 2871441 INFO  (qtp1678053343-18653) [n:127.0.0.1:38210_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard1.replica1, tag=577898264
   [junit4]   2> 2871454 INFO  (qtp1678053343-18653) [n:127.0.0.1:38210_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=halfdeletedcollection_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=137
   [junit4]   2> 2871455 INFO  (qtp882061671-18684) [n:127.0.0.1:51654_solr    
] o.a.s.m.SolrMetricManager Closing metric reporters for 
registry=solr.core.halfdeletedcollection.shard2.replica1, tag=1226174404
   [junit4]   2> 2871462 INFO  (qtp882061671-18684) [n:127.0.0.1:51654_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=halfdeletedcollection_shard2_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=142
   [junit4]   2> 2872167 INFO  (qtp810562917-18674) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={name=halfdeletedcollection&action=DELETE&wt=javabin&version=2} status=0 
QTime=866
   [junit4]   2> 2872169 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2872180 INFO  
(OverseerThreadFactory-4682-thread-5-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
collection
   [junit4]   2> 2872180 INFO  
(OverseerCollectionConfigSetProcessor-97772263390445577-127.0.0.1:51654_solr-n_0000000000)
 [n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2872180 WARN  
(OverseerThreadFactory-4682-thread-5-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CreateCollectionCmd Specified 
replicationFactor of 10 on collection collection is higher than or equal to the 
number of Solr instances currently live or live and part of your 
createNodeSet(4). It's unusual to run two replica of the same slice on the same 
Solr-instance.
   [junit4]   2> 2872181 ERROR 
(OverseerThreadFactory-4682-thread-5-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: collection operation: create 
failed:org.apache.solr.common.SolrException: Cannot create collection 
collection. Value of maxShardsPerNode is 1, and the number of nodes currently 
live or live and part of your createNodeSet is 4. This allows a maximum of 4 to 
be created. Value of numShards is 2 and value of replicationFactor is 10. This 
requires 20 shards to be created (higher than the allowed number)
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:226)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 2872186 INFO  (qtp810562917-18676) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2}
 status=400 QTime=17
   [junit4]   2> 2872186 INFO  
(TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testTooManyReplicas
   [junit4]   2> 2872253 INFO  
(TEST-CollectionsAPIDistributedZkTest.testCollectionsAPI-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testCollectionsAPI
   [junit4]   2> 2872254 INFO  
(TEST-CollectionsAPIDistributedZkTest.testCollectionsAPI-seed#[3BBD3F23147BAA1F])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2872256 INFO  (qtp810562917-18672) [n:127.0.0.1:37108_solr    
] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
async=e2b326e2-4beb-4f6c-a911-73a14a157052&replicationFactor=3&collection.configName=conf2&maxShardsPerNode=4&name=awhollynewcollection_0&action=CREATE&numShards=4&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 2872257 INFO  (qtp810562917-18672) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={async=e2b326e2-4beb-4f6c-a911-73a14a157052&replicationFactor=3&collection.configName=conf2&maxShardsPerNode=4&name=awhollynewcollection_0&action=CREATE&numShards=4&wt=javabin&version=2}
 status=0 QTime=1
   [junit4]   2> 2872268 INFO  
(OverseerThreadFactory-4682-thread-5-processing-n:127.0.0.1:51654_solr) 
[n:127.0.0.1:51654_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
awhollynewcollection_0
   [junit4]   2> 2872268 INFO  
(OverseerCollectionConfigSetProcessor-97772263390445577-127.0.0.1:51654_solr-n_0000000000)
 [n:127.0.0.1:51654_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 2872484 INFO  (zkCallback-2876-thread-5) [    ] 
o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeCreated 
path:/collections/awhollynewcollection_0/state.json] for collection 
[awhollynewcollection_0] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2872664 INFO  (qtp1678053343-18654) [n:127.0.0.1:38210_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600782715&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2}
 status=0 QTime=7
   [junit4]   2> 2872668 INFO  
(parallelCoreAdminExecutor-4683-thread-1-processing-n:127.0.0.1:38210_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598600782715 CREATE) 
[n:127.0.0.1:38210_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600782715&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2872672 INFO  (qtp882061671-18685) [n:127.0.0.1:51654_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600685873&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2}
 status=0 QTime=86
   [junit4]   2> 2872673 INFO  
(parallelCoreAdminExecutor-4686-thread-1-processing-n:127.0.0.1:51654_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598600685873 CREATE) 
[n:127.0.0.1:51654_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600685873&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2872715 INFO  (qtp810562917-18675) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598624111855&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=102
   [junit4]   2> 2872740 INFO  
(parallelCoreAdminExecutor-4688-thread-1-processing-n:127.0.0.1:37108_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598624111855 CREATE) 
[n:127.0.0.1:37108_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598624111855&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2872776 INFO  (qtp882061671-18682) [n:127.0.0.1:51654_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598625970927&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=148
   [junit4]   2> 2872804 INFO  (qtp882061671-18684) [n:127.0.0.1:51654_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598616108132&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2}
 status=0 QTime=68
   [junit4]   2> 2872805 INFO  
(parallelCoreAdminExecutor-4686-thread-3-processing-n:127.0.0.1:51654_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598616108132 CREATE) 
[n:127.0.0.1:51654_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598616108132&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2872824 INFO  (qtp1678053343-18647) [n:127.0.0.1:38210_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600834088&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2}
 status=0 QTime=87
   [junit4]   2> 2872825 INFO  
(parallelCoreAdminExecutor-4683-thread-3-processing-n:127.0.0.1:38210_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598600834088 CREATE) 
[n:127.0.0.1:38210_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598600834088&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2872825 INFO  (qtp810562917-18678) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598664104019&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2}
 status=0 QTime=139
   [junit4]   2> 2872826 INFO  
(parallelCoreAdminExecutor-4688-thread-3-processing-n:127.0.0.1:37108_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598664104019 CREATE) 
[n:127.0.0.1:37108_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598664104019&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard3_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2872840 INFO  (qtp1678053343-18651) [n:127.0.0.1:38210_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598608103697&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2}
 status=0 QTime=176
   [junit4]   2> 2872840 INFO  
(parallelCoreAdminExecutor-4683-thread-2-processing-n:127.0.0.1:38210_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598608103697 CREATE) 
[n:127.0.0.1:38210_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598608103697&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2872912 INFO  (qtp810562917-18677) [n:127.0.0.1:37108_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598632103422&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2}
 status=0 QTime=271
   [junit4]   2> 2872913 INFO  
(parallelCoreAdminExecutor-4688-thread-2-processing-n:127.0.0.1:37108_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598632103422 CREATE) 
[n:127.0.0.1:37108_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598632103422&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2872916 INFO  (qtp397407894-18666) [n:127.0.0.1:45966_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598649805408&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2}
 status=0 QTime=124
   [junit4]   2> 2872917 INFO  
(parallelCoreAdminExecutor-4686-thread-2-processing-n:127.0.0.1:51654_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598625970927 CREATE) 
[n:127.0.0.1:51654_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598625970927&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2872959 INFO  (qtp397407894-18668) [n:127.0.0.1:45966_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598660125484&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2}
 status=0 QTime=5
   [junit4]   2> 2872960 INFO  
(parallelCoreAdminExecutor-4684-thread-1-processing-n:127.0.0.1:45966_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598649805408 CREATE) 
[n:127.0.0.1:45966_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598649805408&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard4_replica1&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2872962 INFO  
(parallelCoreAdminExecutor-4684-thread-1-processing-n:127.0.0.1:45966_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598649805408 CREATE) 
[n:127.0.0.1:45966_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating 
transient cache for 2147483647 transient cores
   [junit4]   2> 2872973 INFO  
(parallelCoreAdminExecutor-4684-thread-2-processing-n:127.0.0.1:45966_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598660125484 CREATE) 
[n:127.0.0.1:45966_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598660125484&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard1_replica2&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2873001 INFO  (qtp397407894-18666) [n:127.0.0.1:45966_solr    
] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598612099620&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 2873002 INFO  
(parallelCoreAdminExecutor-4684-thread-3-processing-n:127.0.0.1:45966_solr 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598612099620 CREATE) 
[n:127.0.0.1:45966_solr    ] o.a.s.h.a.CoreAdminOperation core create command 
async=e2b326e2-4beb-4f6c-a911-73a14a15705230262598612099620&qt=/admin/cores&collection.configName=conf2&newCollection=true&name=awhollynewcollection_0_shard2_replica3&action=CREATE&numShards=4&collection=awhollynewcollection_0&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2873003 INFO  (qtp397407894-18661) [n:127.0.0.1:45966_solr    
] o.a.s.h.a.CoreAdminOperation Checking request status for : 
e2b326e2-4beb-4f6c-a911-73a14a15705230262598660125484
   [junit4]   2> 2873003 INFO  (qtp397407894-18661) [n:127.0.0.1:

[...truncated too long message...]

.java:463)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:77)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 53 more
   [junit4]   2> 5703871 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    
] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44105 44105
   [junit4]   2> 5703871 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[3BBD3F23147BAA1F]-worker) [    
] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_3BBD3F23147BAA1F-001
   [junit4]   2> Apr 11, 2017 5:39:46 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 18 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=RandomSimilarity(queryNorm=false): {}, 
locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=NZ
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_121 (64-bit)/cpus=4,threads=2,free=272222752,total=525336576
   [junit4]   2> NOTE: All tests run in this JVM: [MBeansHandlerTest, 
TestDefaultSearchFieldResource, TestDocSet, SimpleMLTQParserTest, 
SolrIndexConfigTest, TestConfig, TestExtendedDismaxParser, OutputWriterTest, 
TestCorePropertiesReload, TestGroupingSearch, TestRecovery, 
PrimitiveFieldTypeTest, TestSQLHandler, TestQuerySenderNoQuery, 
TestPKIAuthenticationPlugin, TestFileDictionaryLookup, TestTolerantSearch, 
TestReload, OutOfBoxZkACLAndCredentialsProvidersTest, 
TestCloudPseudoReturnFields, SolrMetricManagerTest, 
IgnoreCommitOptimizeUpdateProcessorFactoryTest, StressHdfsTest, 
TestLeaderInitiatedRecoveryThread, VMParamsZkACLAndCredentialsProvidersTest, 
TestPHPSerializedResponseWriter, TestManagedSchemaThreadSafety, TestZkChroot, 
SolrIndexMetricsTest, TestPivotHelperCode, DocValuesMultiTest, 
SortSpecParsingTest, RecoveryZkTest, SecurityConfHandlerTest, 
TestReplicationHandlerBackup, TestFieldCacheSortRandom, LeaderElectionTest, 
TestTolerantUpdateProcessorRandomCloud, LargeFieldTest, 
CurrencyFieldXmlFileTest, TestImplicitCoreProperties, 
TestSchemalessBufferedUpdates, TestIBSimilarityFactory, 
TestSubQueryTransformer, StatsComponentTest, SuggesterTest, 
SpellCheckCollatorWithCollapseTest, CoreAdminCreateDiscoverTest, 
RestartWhileUpdatingTest, HighlighterConfigTest, TestShortCircuitedRequests, 
DebugComponentTest, TestSolrQueryParserDefaultOperatorResource, 
HardAutoCommitTest, BigEndianAscendingWordDeserializerTest, 
TestCloudManagedSchema, RemoteQueryErrorTest, TestConfigSetImmutable, 
TestCollectionAPI, TestReversedWildcardFilterFactory, 
TermVectorComponentDistributedTest, MigrateRouteKeyTest, HdfsNNFailoverTest, 
PreAnalyzedFieldManagedSchemaCloudTest, FullSolrCloudDistribCmdsTest, 
BJQParserTest, FieldAnalysisRequestHandlerTest, TestExactStatsCache, 
AnalyticsMergeStrategyTest, TestComponentsName, BasicDistributedZk2Test, 
TestRawTransformer, TestMiniSolrCloudCluster, RequestHandlersTest, 
TestCoreDiscovery, TestSolrCLIRunExample, TestNoOpRegenerator, 
FacetPivotSmallTest, QueryResultKeyTest, 
DistributedQueryComponentCustomSortTest, SolrCoreCheckLockOnStartupTest, 
TestSolrCloudWithKerberosAlt, BlockJoinFacetSimpleTest, DateFieldTest, 
TestSolrDeletionPolicy1, TestApiFramework, TestMinMaxOnMultiValuedField, 
OverseerTaskQueueTest, SubstringBytesRefFilterTest, TestManagedResource, 
TestSolrQueryParser, TestSimpleQParserPlugin, TestMaxScoreQueryParser, 
LukeRequestHandlerTest, DistributedFacetExistsSmallTest, DeleteShardTest, 
SharedFSAutoReplicaFailoverUtilsTest, TestJavabinTupleStreamParser, 
JsonValidatorTest, TestSchemaSimilarityResource, SolrSlf4jReporterTest, 
TestSolrJ, TestRandomCollapseQParserPlugin, TestComplexPhraseLeadingWildcard, 
DistributedVersionInfoTest, StatelessScriptUpdateProcessorFactoryTest, 
SimpleFacetsTest, AsyncCallRequestStatusResponseTest, TestStressUserVersions, 
TestCollationField, TestFieldTypeResource, RAMDirectoryFactoryTest, 
TestBlobHandler, UpdateParamsTest, OverseerTest, 
UninvertDocValuesMergePolicyTest, UnloadDistributedZkTest, 
CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, 
LeaderElectionIntegrationTest, TestReplicationHandler, AutoCommitTest, 
TestHashPartitioner, DistributedSpellCheckComponentTest, TestStressReorder, 
TestStressVersions, TestRangeQuery, TestQuerySenderListener, 
TestSolrIndexConfig, CopyFieldTest, ResponseLogComponentTest, BadComponentTest, 
TestMergePolicyConfig, TestSolrDeletionPolicy2, TestBinaryField, 
TestElisionMultitermQuery, ExternalFileFieldSortTest, TestSolrCoreProperties, 
TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, 
TestPhraseSuggestions, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, 
TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, 
TestJmxMonitoredMap, ResourceLoaderTest, TestFastOutputStream, 
URLClassifyProcessorTest, PreAnalyzedFieldTest, PrimUtilsTest, 
SpellingQueryConverterTest, DOMUtilTest, TestUtils, SliceStateTest, 
SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, 
DistributedMLTComponentTest, CursorPagingTest, SolrTestCaseJ4Test, 
TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, 
TestSimpleTrackingShardHandler, TestEmbeddedSolrServerSchemaAPI, 
ActionThrottleTest, AliasIntegrationTest, AssignTest, 
BaseCdcrDistributedZkTest, CdcrBootstrapTest, CloudExitableDirectoryReaderTest, 
CollectionTooManyReplicasTest, CollectionsAPISolrJTest, 
DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteStatusTest, 
DistribCursorPagingTest, DocValuesNotIndexedTest, 
LeaderFailoverAfterPartitionTest, MoveReplicaTest, 
OverriddenZkACLAndCredentialsProvidersTest, 
OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, 
SaslZkACLProviderTest, ShardSplitTest, HdfsChaosMonkeyNothingIsSafeTest, 
HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [540/704 (4!)] on J2 in 149.18s, 21 tests, 1 failure <<< 
FAILURES!

[...truncated 43816 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to