Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.4/3/

9 tests failed.
FAILED:  
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete 
@source_collection:shard2
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:19FF8DD32B598908]:0)
        at 
org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:795)
        at 
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:526)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ 
target_collection
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:CE8766ED53194D61]:0)
        at 
org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:271)
        at 
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:595)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<50> but was:<20>
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:262)
        ... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete 
@source_collection:shard1
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:729D6894CC7383F9]:0)
        at 
org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:795)
        at 
org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps(CdcrReplicationDistributedZkTest.java:463)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
  at sun.reflect.GeneratedConstructorAccessor182.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)  at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
        at sun.reflect.GeneratedConstructorAccessor182.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)
        at 
org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)
        at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([6EE5F47783408824]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
        at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:E6B1CBAD2DBCE5DC]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNotNull(Assert.java:526)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:159)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
        at 
org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
        at 
org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig

Error Message:
The max direct memory is likely too low.  Either increase it (by adding 
-XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup 
args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~134.217728 MB.

Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low.  Either 
increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to 
your containers startup args) or disable direct allocation using 
solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you 
are putting the block cache on the heap, your java heap size might not be large 
enough. Failed allocating ~134.217728 MB.
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:994A3D5C7EC9620F]:0)
        at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:307)
        at 
org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:283)
        at 
org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:223)
        at 
org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:108)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
        at java.nio.Bits.reserveMemory(Bits.java:693)
        at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
        at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
        at 
org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:68)
        at 
org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:305)
        ... 42 more


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Expected to see collection awhollynewcollection_0 null Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/2)={
   "replicationFactor":"2",   "shards":{     "shard1":{       
"range":"80000000-ffffffff",       "state":"active",       "replicas":{}},     
"shard2":{       "range":"0-7fffffff",       "state":"active",       
"replicas":{}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"2",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected to see collection awhollynewcollection_0
null
Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/2)={
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:269080C38573A7B1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:265)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:496)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
KeeperErrorCode = NoNode for /collections/withconfigset2

Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode 
for /collections/withconfigset2
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:439BBB2D74643228]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testSpecificConfigsets(CollectionsAPIDistributedZkTest.java:425)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
        at 
__randomizedtesting.SeedInfo.seed([6EE5F47783408824:E6B1CBAD2DBCE5DC]:0)
        at org.junit.Assert.fail(Assert.java:92)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertTrue(Assert.java:54)
        at 
org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.test(HdfsWriteToMultipleCollectionsTest.java:137)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12623 lines...]
   [junit4] Suite: org.apache.solr.cloud.CdcrReplicationDistributedZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/init-core-data-001
   [junit4]   2> 1763641 INFO  
(SUITE-CdcrReplicationDistributedZkTest-seed#[6EE5F47783408824]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1763641 INFO  
(SUITE-CdcrReplicationDistributedZkTest-seed#[6EE5F47783408824]-worker) [    ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1763643 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1763643 INFO  (Thread-263131) [    ] o.a.s.c.ZkTestServer 
client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1763643 INFO  (Thread-263131) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 1763765 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.ZkTestServer start zk server on port:37201
   [junit4]   2> 1763814 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 1763815 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 1763817 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1763818 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 1763819 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 1763820 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 1763822 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1763823 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1763824 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1763825 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1763826 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 1763830 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testReplicationAfterRestart
   [junit4]   2> 1763947 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/cores/collection1
   [junit4]   2> 1763948 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1763949 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3470e95b{/,null,AVAILABLE}
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.AbstractConnector Started 
ServerConnector@163d0bc3{HTTP/1.1,[http/1.1]}{127.0.0.1:52802}
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server Started @1767369ms
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, 
hostContext=/, hostPort=52802, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/cores}
   [junit4]   2> 1763950 ERROR 
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ 
version 6.4.0
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1763950 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-01-17T11:25:27.456Z
   [junit4]   2> 1763990 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1763990 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/solr.xml
   [junit4]   2> 1763996 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1763997 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37201/solr
   [junit4]   2> 1764125 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:52802_    ] o.a.s.c.OverseerElectionContext I am going to be the 
leader 127.0.0.1:52802_
   [junit4]   2> 1764127 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:52802_    ] o.a.s.c.Overseer Overseer 
(id=97298174912102404-127.0.0.1:52802_-n_0000000000) starting
   [junit4]   2> 1764156 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:52802_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:52802_
   [junit4]   2> 1764163 INFO  
(zkCallback-556-thread-1-processing-n:127.0.0.1:52802_) [n:127.0.0.1:52802_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1764307 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:52802_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/cores
   [junit4]   2> 1764307 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:52802_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1764333 INFO  
(OverseerStateUpdate-97298174912102404-127.0.0.1:52802_-n_0000000000) 
[n:127.0.0.1:52802_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard2
   [junit4]   2> 1765335 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.c.CreateCollectionCmd Only one config set found in zk - using it:conf1
   [junit4]   2> 1765342 WARN  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.c.Config Beginning 
with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1765342 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.4.0
   [junit4]   2> 1765352 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1765506 WARN  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1765507 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1765526 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection 
tmp_collection
   [junit4]   2> 1765527 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-001/cores/collection1/data/]
   [junit4]   2> 1765527 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.c.JmxMonitoredMap 
JMX monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4356e3da
   [junit4]   2> 1765536 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.7034694537142971]
   [junit4]   2> 1765651 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 1765651 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1765652 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1765652 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1765655 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: 
org.apache.lucene.index.MockRandomMergePolicy@27f115fd
   [junit4]   2> 1765656 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@4b9d0afe[collection1] main]
   [junit4]   2> 1765657 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1765658 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1765658 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1765667 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.h.CdcrBufferStateManager Created znode 
/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 1765673 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.h.CdcrProcessStateManager Created znode 
/collections/tmp_collection/cdcr/state/process
   [junit4]   2> 1765701 INFO  
(searcherExecutor-1973-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@4b9d0afe[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1765706 INFO  
(coreLoadExecutor-1972-thread-1-processing-n:127.0.0.1:52802_) 
[n:127.0.0.1:52802_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog Could 
not find max version in index or recent updates, using new clock 
1556770800755802112
   [junit4]   2> 1765713 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to 
continue.
   [junit4]   2> 1765713 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try 
and sync
   [junit4]   2> 1765713 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:52802/collection1/
   [junit4]   2> 1765713 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1765713 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52802/collection1/ has no 
replicas
   [junit4]   2> 1765718 INFO  
(zkCallback-556-thread-3-processing-n:127.0.0.1:52802_) [n:127.0.0.1:52802_    
] o.a.s.h.CdcrLeaderStateManager Received new leader state @ 
tmp_collection:shard2
   [junit4]   2> 1765720 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:52802/collection1/ shard2
   [junit4]   2> 1765916 INFO  
(coreZkRegister-1965-thread-1-processing-n:127.0.0.1:52802_ x:collection1 
c:tmp_collection) [n:127.0.0.1:52802_ c:tmp_collection s:shard2 r:core_node1 
x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1766317 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/cores/collection1
   [junit4]   2> 1766320 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1766333 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3aa119fa{/,null,AVAILABLE}
   [junit4]   2> 1766333 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.AbstractConnector Started 
ServerConnector@35cd505b{HTTP/1.1,[http/1.1]}{127.0.0.1:49817}
   [junit4]   2> 1766333 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server Started @1769753ms
   [junit4]   2> 1766333 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, 
hostContext=/, hostPort=49817, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/cores}
   [junit4]   2> 1766333 ERROR 
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 1766336 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ 
version 6.4.0
   [junit4]   2> 1766336 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 1766336 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1766336 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-01-17T11:25:29.842Z
   [junit4]   2> 1766350 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1766350 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/solr.xml
   [junit4]   2> 1766359 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1766360 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37201/solr
   [junit4]   2> 1766376 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:49817_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1766385 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:49817_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:49817_
   [junit4]   2> 1766388 INFO  
(zkCallback-556-thread-1-processing-n:127.0.0.1:52802_) [n:127.0.0.1:52802_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1766422 INFO  
(zkCallback-562-thread-1-processing-n:127.0.0.1:49817_) [n:127.0.0.1:49817_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1766530 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:49817_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/cores
   [junit4]   2> 1766530 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:49817_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1766536 INFO  
(OverseerStateUpdate-97298174912102404-127.0.0.1:52802_-n_0000000000) 
[n:127.0.0.1:52802_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 1767580 WARN  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.c.Config Beginning 
with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1767581 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.4.0
   [junit4]   2> 1767596 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1767692 WARN  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1767693 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1767717 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection 
tmp_collection
   [junit4]   2> 1767718 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-002/cores/collection1/data/]
   [junit4]   2> 1767718 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.c.JmxMonitoredMap 
JMX monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4356e3da
   [junit4]   2> 1767719 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.7034694537142971]
   [junit4]   2> 1767767 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 1767767 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1767769 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1767769 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1767776 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: 
org.apache.lucene.index.MockRandomMergePolicy@52a3713b
   [junit4]   2> 1767779 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@1c1f53f4[collection1] main]
   [junit4]   2> 1767784 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1767786 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1767788 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1767798 INFO  
(coreLoadExecutor-1984-thread-1-processing-n:127.0.0.1:49817_) 
[n:127.0.0.1:49817_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog Could 
not find max version in index or recent updates, using new clock 
1556770802949423104
   [junit4]   2> 1767802 INFO  
(searcherExecutor-1985-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@1c1f53f4[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1767817 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to 
continue.
   [junit4]   2> 1767818 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try 
and sync
   [junit4]   2> 1767818 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:49817/collection1/
   [junit4]   2> 1767818 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1767818 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:49817/collection1/ has no 
replicas
   [junit4]   2> 1767824 INFO  
(zkCallback-562-thread-1-processing-n:127.0.0.1:49817_) [n:127.0.0.1:49817_    
] o.a.s.h.CdcrLeaderStateManager Received new leader state @ 
tmp_collection:shard1
   [junit4]   2> 1767835 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:49817/collection1/ shard1
   [junit4]   2> 1767936 INFO  
(coreZkRegister-1979-thread-1-processing-n:127.0.0.1:49817_ x:collection1 
c:tmp_collection) [n:127.0.0.1:49817_ c:tmp_collection s:shard1 r:core_node2 
x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1768055 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/cores/collection1
   [junit4]   2> 1768058 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1768065 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@261d4086{/,null,AVAILABLE}
   [junit4]   2> 1768065 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.AbstractConnector Started 
ServerConnector@44958006{HTTP/1.1,[http/1.1]}{127.0.0.1:44348}
   [junit4]   2> 1768065 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.e.j.s.Server Started @1771485ms
   [junit4]   2> 1768065 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard3, 
hostContext=/, hostPort=44348, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/cores}
   [junit4]   2> 1768066 ERROR 
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging 
may be missing or incomplete.
   [junit4]   2> 1768067 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ 
version 6.4.0
   [junit4]   2> 1768067 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on 
port null
   [junit4]   2> 1768067 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1768067 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-01-17T11:25:31.573Z
   [junit4]   2> 1768073 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1768073 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/solr.xml
   [junit4]   2> 1768079 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1768081 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37201/solr
   [junit4]   2> 1768114 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:44348_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 1768119 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:44348_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:44348_
   [junit4]   2> 1768120 INFO  
(zkCallback-562-thread-1-processing-n:127.0.0.1:49817_) [n:127.0.0.1:49817_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1768122 INFO  
(zkCallback-556-thread-2-processing-n:127.0.0.1:52802_) [n:127.0.0.1:52802_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1768145 INFO  
(zkCallback-568-thread-1-processing-n:127.0.0.1:44348_) [n:127.0.0.1:44348_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1768203 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:44348_    ] o.a.s.c.CorePropertiesLocator Found 1 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/cores
   [junit4]   2> 1768203 INFO  
(TEST-CdcrReplicationDistributedZkTest.testReplicationAfterRestart-seed#[6EE5F47783408824])
 [n:127.0.0.1:44348_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1768212 INFO  
(OverseerStateUpdate-97298174912102404-127.0.0.1:52802_-n_0000000000) 
[n:127.0.0.1:52802_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard2
   [junit4]   2> 1769248 WARN  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.c.Config Beginning 
with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1769248 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.4.0
   [junit4]   2> 1769258 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1769331 WARN  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1769332 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 1769344 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection 
tmp_collection
   [junit4]   2> 1769347 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_6EE5F47783408824-001/jetty-003/cores/collection1/data/]
   [junit4]   2> 1769347 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.c.JmxMonitoredMap 
JMX monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@4356e3da
   [junit4]   2> 1769348 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.7034694537142971]
   [junit4]   2> 1769363 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 1769364 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1769364 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1769364 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1769372 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: 
org.apache.lucene.index.MockRandomMergePolicy@42e9a45
   [junit4]   2> 1769375 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@7fc14a08[collection1] main]
   [junit4]   2> 1769377 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1769378 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1769378 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1769380 INFO  
(searcherExecutor-1997-thread-1-processing-n:127.0.0.1:44348_ x:collection1 
c:tmp_collection) [n:127.0.0.1:44348_ c:tmp_collection   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@7fc14a08[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1769385 INFO  
(coreLoadExecutor-1996-thread-1-processing-n:127.0.0.1:44348_) 
[n:127.0.0.1:44348_ c:tmp_collection   x:collection1] o.a.s.u.UpdateLog Could 
not find max version in index or recent updates, using new clock 
1556770804613513216
   [junit4]   2> 1769401 INFO  
(coreZkRegister-1991-thread-1-processing-n:127.0.0.1:44348_ x:collection1 
c:tmp_collection) [n:127.0.0.1:44348_ c:tmp_collection s:shard2 r:core_node3 
x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 1769416 INFO  
(updateExecutor-565-thread-1-processing-n:127.0.0.1:44348_ x:collection1 
s:shard2 c:tmp_collection r:core_node3) [n:127.0.0.1:44348_ c:tmp_collection 
s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running 
recovery
   [junit4]   2> 1769417 INFO  
(recoveryExecutor-566-thread-1-processing-n:127.0.0.1:44348_ x:collection1 
s:shard2 c:tmp_collection r:core_node3) [n:127.0.0.1:44348_ c:tmp_collection 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery 
process. recoveringAfterStartup=true
   [junit4]   2> 1769418 INFO  
(recoveryExecutor-566-thread-1-processing-n:127.0.0.1:44348_ x:collection1 
s:shard2 c:tmp_collection r:core_node3) [n:127.0.0.1:44348_ c:tmp_collection 
s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### 
startupVersions=[[]]
   [junit4]   2> 1769418 INFO  (recoveryExecutor-566-thread-1-

[...truncated too long message...]

/temp/solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest_6EE5F47783408824-001/tempDir-001/hdfsBaseDir/data/data2/]]
  heartbeating to localhost/127.0.0.1:36756) [    ] o.a.h.h.s.d.DataNode Ending 
block pool service for: Block pool BP-1632051331-10.41.0.5-1484655261156 
(Datanode Uuid d2deb2ce-a884-4719-8fe3-60f7ef3d27ba) service to 
localhost/127.0.0.1:36756
   [junit4]   2> 4734121 INFO  
(SUITE-HdfsWriteToMultipleCollectionsTest-seed#[6EE5F47783408824]-worker) [    
] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 4734223 ERROR 
(SUITE-HdfsWriteToMultipleCollectionsTest-seed#[6EE5F47783408824]-worker) [    
] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
   [junit4]   2> java.lang.reflect.InvocationTargetException
   [junit4]   2>        at 
sun.reflect.GeneratedMethodAccessor154.invoke(Unknown Source)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2>        at 
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2>        at 
com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest.teardownClass(HdfsWriteToMultipleCollectionsTest.java:73)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 53 more
   [junit4]   2> 4734225 INFO  
(SUITE-HdfsWriteToMultipleCollectionsTest-seed#[6EE5F47783408824]-worker) [    
] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsWriteToMultipleCollectionsTest_6EE5F47783408824-001
   [junit4]   2> Jan 17, 2017 12:14:57 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 35 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, 
sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=sr-Latn, 
timezone=America/Regina
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_102 (64-bit)/cpus=4,threads=2,free=207909144,total=533200896
   [junit4]   2> NOTE: All tests run in this JVM: [QueryElevationComponentTest, 
TestSolrCloudWithSecureImpersonation, TestRandomFaceting, TestSQLHandler, 
TestCryptoKeys, HdfsBasicDistributedZkTest, DateFieldTest, TestFieldSortValues, 
TestQuerySenderNoQuery, TestReplicationHandler, TestDistributedSearch, 
PeerSyncWithIndexFingerprintCachingTest, TestComponentsName, ReturnFieldsTest, 
TestSolrDeletionPolicy2, DefaultValueUpdateProcessorTest, 
ChaosMonkeyNothingIsSafeTest, SimpleCollectionCreateDeleteTest, 
TestExceedMaxTermLength, TestSizeLimitedDistributedMap, 
CdcrReplicationDistributedZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, 
TestFunctionQuery, SolrXmlInZkTest, TestCustomDocTransformer, PeerSyncTest, 
TestCloudRecovery, ActionThrottleTest, TestDistribIDF, TestFaceting, 
RuleEngineTest, TestSSLRandomization, DistribDocExpirationUpdateProcessorTest, 
HdfsRecoveryZkTest, AlternateDirectoryTest, 
TestPerFieldSimilarityWithDefaultOverride, HttpPartitionTest, ReplaceNodeTest, 
OutputWriterTest, DocExpirationUpdateProcessorFactoryTest, 
AnalysisAfterCoreReloadTest, TestSchemaResource, CollectionTooManyReplicasTest, 
TestBinaryResponseWriter, CloudMLTQParserTest, DisMaxRequestHandlerTest, 
TriLevelCompositeIdRoutingTest, BlockJoinFacetRandomTest, 
DocumentAnalysisRequestHandlerTest, TestDynamicFieldResource, 
DistanceUnitsTest, QueryResultKeyTest, TestUniqueKeyFieldResource, 
TestEmbeddedSolrServerConstructors, AsyncCallRequestStatusResponseTest, 
TestAddFieldRealTimeGet, DateRangeFieldTest, TestGraphMLResponseWriter, 
ConjunctionSolrSpellCheckerTest, TestLocalFSCloudBackupRestore, 
TestIBSimilarityFactory, TestRestManager, TestRandomFlRTGCloud, BasicZkTest, 
BasicDistributedZkTest, ClassificationUpdateProcessorIntegrationTest, 
DistributedFacetPivotLargeTest, TestSolrConfigHandlerConcurrent, 
QueryEqualityTest, CoreMergeIndexesAdminHandlerTest, 
TestSolrCloudWithHadoopAuthPlugin, HdfsWriteToMultipleCollectionsTest]
   [junit4] Completed [641/680 (6!)] on J0 in 46.30s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 62610 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to