Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.4/20/

4 tests failed.
FAILED:  
org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored_idx

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 
seconds
        at 
__randomizedtesting.SeedInfo.seed([5A047B097F177FD2:50232A290061DC85]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
        at 
org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.waitForRecoveriesToFinish(TestStressCloudBlindAtomicUpdates.java:459)
        at 
org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:304)
        at 
org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored_idx(TestStressCloudBlindAtomicUpdates.java:214)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test

Error Message:
Mismatch in counts between replicas

Stack Trace:
java.lang.AssertionError: Mismatch in counts between replicas
        at 
__randomizedtesting.SeedInfo.seed([5A047B097F177FD2:D25044D3D1EB122A]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.RecoveryZkTest.assertShardConsistency(RecoveryZkTest.java:143)
        at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:126)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
KeeperErrorCode = NoNode for /collections/withconfigset2

Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode 
for /collections/withconfigset2
        at 
__randomizedtesting.SeedInfo.seed([5A047B097F177FD2:777A34538833C5DE]:0)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
        at 
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
        at 
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at 
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testSpecificConfigsets(CollectionsAPIDistributedZkTest.java:425)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Expected to see collection awhollynewcollection_0 null Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/2)={
   "replicationFactor":"1",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"1",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected to see collection awhollynewcollection_0
null
Last available state: 
DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/2)={
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([5A047B097F177FD2:12710FBD79245047]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:265)
        at 
org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:496)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11720 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/init-core-data-001
   [junit4]   2> 894981 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 894982 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001
   [junit4]   2> 894983 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 894985 INFO  (Thread-1195) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 894985 INFO  (Thread-1195) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 895085 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.c.ZkTestServer start zk server on port:45280
   [junit4]   2> 895095 INFO  (jetty-launcher-506-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 895096 INFO  (jetty-launcher-506-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 895096 INFO  (jetty-launcher-506-thread-3) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 895097 INFO  (jetty-launcher-506-thread-4) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 895098 INFO  (jetty-launcher-506-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@396d3163{/solr,null,AVAILABLE}
   [junit4]   2> 895100 INFO  (jetty-launcher-506-thread-2) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@5d2d33ed{SSL,[ssl, 
http/1.1]}{127.0.0.1:58634}
   [junit4]   2> 895100 INFO  (jetty-launcher-506-thread-2) [    ] 
o.e.j.s.Server Started @898629ms
   [junit4]   2> 895100 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=58634}
   [junit4]   2> 895101 ERROR (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 895101 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.4.2
   [junit4]   2> 895101 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 895101 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 895101 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T04:25:31.187Z
   [junit4]   2> 895101 INFO  (jetty-launcher-506-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@e8c7fce{/solr,null,AVAILABLE}
   [junit4]   2> 895102 INFO  (jetty-launcher-506-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@65bc9f6f{SSL,[ssl, 
http/1.1]}{127.0.0.1:54331}
   [junit4]   2> 895102 INFO  (jetty-launcher-506-thread-1) [    ] 
o.e.j.s.Server Started @898631ms
   [junit4]   2> 895102 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=54331}
   [junit4]   2> 895103 ERROR (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.4.2
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@4141ffc1{/solr,null,AVAILABLE}
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T04:25:31.189Z
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@179c09f3{SSL,[ssl, 
http/1.1]}{127.0.0.1:46765}
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.e.j.s.Server Started @898632ms
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=46765}
   [junit4]   2> 895103 ERROR (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.4.2
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 895103 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T04:25:31.189Z
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@4f807e19{/solr,null,AVAILABLE}
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@58256c54{SSL,[ssl, 
http/1.1]}{127.0.0.1:51242}
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.e.j.s.Server Started @898641ms
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=51242}
   [junit4]   2> 895112 ERROR (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.4.2
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 895112 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 895113 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-03-03T04:25:31.199Z
   [junit4]   2> 895113 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 895113 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 895128 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 895133 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 895133 INFO  (jetty-launcher-506-thread-4) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45280/solr
   [junit4]   2> 895138 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 895139 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 895150 INFO  (jetty-launcher-506-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45280/solr
   [junit4]   2> 895150 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 895151 INFO  (jetty-launcher-506-thread-3) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45280/solr
   [junit4]   2> 895152 INFO  (jetty-launcher-506-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45280/solr
   [junit4]   2> 895176 INFO  (jetty-launcher-506-thread-3) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:51242_solr
   [junit4]   2> 895177 INFO  (jetty-launcher-506-thread-4) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:46765_solr
   [junit4]   2> 895178 INFO  (jetty-launcher-506-thread-1) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:54331_solr
   [junit4]   2> 895178 INFO  (jetty-launcher-506-thread-3) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.Overseer Overseer 
(id=97551327629934602-127.0.0.1:51242_solr-n_0000000000) starting
   [junit4]   2> 895179 INFO  
(zkCallback-527-thread-1-processing-n:127.0.0.1:58634_solr) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 895180 INFO  
(zkCallback-525-thread-1-processing-n:127.0.0.1:54331_solr) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 895180 INFO  
(zkCallback-523-thread-1-processing-n:127.0.0.1:46765_solr) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 895180 INFO  
(zkCallback-526-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 895182 INFO  
(zkCallback-523-thread-1-processing-n:127.0.0.1:46765_solr) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 895186 INFO  
(zkCallback-526-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 895186 INFO  
(zkCallback-525-thread-1-processing-n:127.0.0.1:54331_solr) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 895187 INFO  (jetty-launcher-506-thread-2) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:58634_solr
   [junit4]   2> 895189 INFO  
(zkCallback-523-thread-1-processing-n:127.0.0.1:46765_solr) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 895193 INFO  
(zkCallback-525-thread-1-processing-n:127.0.0.1:54331_solr) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 895198 INFO  
(zkCallback-526-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 895206 INFO  
(zkCallback-527-thread-1-processing-n:127.0.0.1:58634_solr) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 895206 INFO  
(zkCallback-527-thread-2-processing-n:127.0.0.1:58634_solr) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (3)
   [junit4]   2> 895219 INFO  (jetty-launcher-506-thread-2) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node2/.
   [junit4]   2> 895222 INFO  (jetty-launcher-506-thread-3) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:51242_solr
   [junit4]   2> 895223 INFO  
(zkCallback-523-thread-1-processing-n:127.0.0.1:46765_solr) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 895223 INFO  
(zkCallback-525-thread-1-processing-n:127.0.0.1:54331_solr) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 895225 INFO  (jetty-launcher-506-thread-1) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node1/.
   [junit4]   2> 895226 INFO  
(zkCallback-527-thread-2-processing-n:127.0.0.1:58634_solr) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 895227 INFO  
(zkCallback-526-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (3) -> (4)
   [junit4]   2> 895230 INFO  (jetty-launcher-506-thread-4) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node4/.
   [junit4]   2> 895301 INFO  (jetty-launcher-506-thread-3) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node3/.
   [junit4]   2> 895468 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 895469 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45280/solr ready
   [junit4]   2> 896035 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 896555 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 896699 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 896718 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 896769 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 896829 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_50300_hdfs____lkt50i/webapp
   [junit4]   2> 897625 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:50300
   [junit4]   2> 898216 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 898220 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 898229 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_36004_datanode____.8a2t2g/webapp
   [junit4]   2> 898624 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36004
   [junit4]   2> 898904 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 898905 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log jetty-6.1.26
   [junit4]   2> 898932 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_39628_datanode____8tjni5/webapp
   [junit4]   2> 899356 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[5A047B097F177FD2]-worker) [    
] o.m.log Started 
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39628
   [junit4]   2> 900192 INFO  (IPC Server handler 3 on 35454) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-f4042a0c-0aa5-4cb2-993d-4a5448e6b031 node 
DatanodeRegistration(127.0.0.1:32824, 
datanodeUuid=c21b506e-b4bf-484b-b075-1306c268af9b, infoPort=35294, 
infoSecurePort=0, ipcPort=60084, 
storageInfo=lv=-56;cid=testClusterID;nsid=654144099;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 6 msecs
   [junit4]   2> 900192 INFO  (IPC Server handler 3 on 35454) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-e0b721cd-8761-473e-a849-680d1e8e590f node 
DatanodeRegistration(127.0.0.1:32824, 
datanodeUuid=c21b506e-b4bf-484b-b075-1306c268af9b, infoPort=35294, 
infoSecurePort=0, ipcPort=60084, 
storageInfo=lv=-56;cid=testClusterID;nsid=654144099;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 900192 INFO  (IPC Server handler 4 on 35454) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-79278be5-849e-4b10-a7a3-ddbaa343499e node 
DatanodeRegistration(127.0.0.1:50821, 
datanodeUuid=4cc57069-c0fb-4a5a-b0ed-0a4e5feb5b87, infoPort=35375, 
infoSecurePort=0, ipcPort=52751, 
storageInfo=lv=-56;cid=testClusterID;nsid=654144099;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 900197 INFO  (IPC Server handler 4 on 35454) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-ce4bf9e8-636e-49e4-a30f-462698326a18 node 
DatanodeRegistration(127.0.0.1:50821, 
datanodeUuid=4cc57069-c0fb-4a5a-b0ed-0a4e5feb5b87, infoPort=35375, 
infoSecurePort=0, ipcPort=52751, 
storageInfo=lv=-56;cid=testClusterID;nsid=654144099;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 900285 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testMissingRequiredParameters
   [junit4]   2> 900285 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 900344 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 900344 ERROR (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Missing 
required parameter: name
   [junit4]   2>        at 
org.apache.solr.common.params.RequiredSolrParams.getParams(RequiredSolrParams.java:90)
   [junit4]   2>        at 
org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:461)
   [junit4]   2>        at 
org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:478)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler$CollectionOperation.lambda$static$0(CollectionsHandler.java:383)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler$CollectionOperation.execute(CollectionsHandler.java:877)
   [junit4]   2>        at 
org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:209)
   [junit4]   2>        at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:166)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
   [junit4]   2>        at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
   [junit4]   2>        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2>        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2>        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2>        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2>        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2>        at 
org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2>        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2>        at 
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
   [junit4]   2>        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2>        at 
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2>        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2>        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 900345 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=CREATE&numShards=2&wt=javabin&version=2} status=400 QTime=1
   [junit4]   2> 900346 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testMissingRequiredParameters
   [junit4]   2> 900388 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testMaxNodesPerShard
   [junit4]   2> 900389 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 900394 INFO  (qtp1243735339-3828) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 900400 INFO  
(OverseerThreadFactory-1966-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
oversharded
   [junit4]   2> 900401 ERROR 
(OverseerThreadFactory-1966-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: oversharded operation: create 
failed:org.apache.solr.common.SolrException: Cannot create collection 
oversharded. Value of maxShardsPerNode is 1, and the number of nodes currently 
live or live and part of your createNodeSet is 4. This allows a maximum of 4 to 
be created. Value of numShards is 3 and value of replicationFactor is 2. This 
requires 6 shards to be created (higher than the allowed number)
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 900405 INFO  (qtp1243735339-3828) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2}
 status=400 QTime=11
   [junit4]   2> 900406 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testMaxNodesPerShard
   [junit4]   2> 900459 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateNodeSet
   [junit4]   2> 900459 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 900462 INFO  (qtp1243735339-3829) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf&name=nodeset_collection&action=CREATE&numShards=2&createNodeSet=https://127.0.0.1:46765/solr,https://127.0.0.1:51242/solr&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 900463 INFO  
(OverseerThreadFactory-1966-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
nodeset_collection
   [junit4]   2> 900463 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 900464 WARN  
(OverseerThreadFactory-1966-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd It is unusual to 
create a collection (nodeset_collection) without cores.
   [junit4]   2> 900571 INFO  (qtp1243735339-3829) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 900572 INFO  (qtp1243735339-3829) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf&name=nodeset_collection&action=CREATE&numShards=2&createNodeSet=https://127.0.0.1:46765/solr,https://127.0.0.1:51242/solr&wt=javabin&version=2}
 status=0 QTime=110
   [junit4]   2> 900572 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testCreateNodeSet
   [junit4]   2> 900615 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreationAndDeletion
   [junit4]   2> 900616 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 900618 INFO  (qtp1243735339-3832) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params 
name=nodeset_collection&action=DELETE&wt=javabin&version=2 and 
sendToOCPQueue=true
   [junit4]   2> 900620 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 900625 INFO  
(OverseerThreadFactory-1966-thread-3-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing 
Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 901330 INFO  (qtp1243735339-3832) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={name=nodeset_collection&action=DELETE&wt=javabin&version=2} status=0 
QTime=712
   [junit4]   2> 901332 INFO  (qtp1243735339-3833) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=1&collection.configName=conf&name=created_and_deleted&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 901334 INFO  
(OverseerThreadFactory-1966-thread-4-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
created_and_deleted
   [junit4]   2> 901334 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 901597 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=created_and_deleted_shard1_replica1&action=CREATE&numShards=1&collection=created_and_deleted&shard=shard1&wt=javabin&version=2
   [junit4]   2> 901704 INFO  
(zkCallback-526-thread-1-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/created_and_deleted/state.json] for collection 
[created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 902612 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.2
   [junit4]   2> 902620 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.IndexSchema [created_and_deleted_shard1_replica1] Schema name=minimal
   [junit4]   2> 902622 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 902623 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'created_and_deleted_shard1_replica1' 
using configuration from collection created_and_deleted
   [junit4]   2> 902639 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35454/data
   [junit4]   2> 902639 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 902640 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.SolrCore [[created_and_deleted_shard1_replica1] ] Opening new SolrCore 
at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node3/created_and_deleted_shard1_replica1],
 dataDir=[hdfs://localhost:35454/data/created_and_deleted/core_node1/data/]
   [junit4]   2> 902641 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data/snapshot_metadata
   [junit4]   2> 902665 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 902665 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 902665 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 904197 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 904203 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data
   [junit4]   2> 904233 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data/index
   [junit4]   2> 904242 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 904242 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 904242 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 904255 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 904483 INFO  (IPC Server handler 2 on 35454) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:50821 is 
added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-f4042a0c-0aa5-4cb2-993d-4a5448e6b031:NORMAL:127.0.0.1:32824|RBW],
 
ReplicaUC[[DISK]DS-79278be5-849e-4b10-a7a3-ddbaa343499e:NORMAL:127.0.0.1:50821|RBW]]}
 size 71
   [junit4]   2> 904484 INFO  (IPC Server handler 4 on 35454) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:32824 is 
added to blk_1073741825_1001 size 71
   [junit4]   2> 904935 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.UpdateHandler Using UpdateLog implementation: 
org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 904935 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 904935 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 904966 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 904966 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 905098 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@4ce248f[created_and_deleted_shard1_replica1] main]
   [junit4]   2> 905099 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 905099 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 905101 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 905101 INFO  
(searcherExecutor-1973-thread-1-processing-n:127.0.0.1:51242_solr 
x:created_and_deleted_shard1_replica1 s:shard1 c:created_and_deleted) 
[n:127.0.0.1:51242_solr c:created_and_deleted s:shard1  
x:created_and_deleted_shard1_replica1] o.a.s.c.SolrCore 
[created_and_deleted_shard1_replica1] Registered new searcher 
Searcher@4ce248f[created_and_deleted_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 905101 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.u.UpdateLog Could not find max version in index or recent updates, using 
new clock 1560821252685299712
   [junit4]   2> 905111 INFO  
(zkCallback-526-thread-3-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/created_and_deleted/state.json] for collection 
[created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 905112 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 905112 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 905112 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.SyncStrategy Sync replicas to 
https://127.0.0.1:51242/solr/created_and_deleted_shard1_replica1/
   [junit4]   2> 905113 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 905113 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.SyncStrategy 
https://127.0.0.1:51242/solr/created_and_deleted_shard1_replica1/ has no 
replicas
   [junit4]   2> 905115 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
https://127.0.0.1:51242/solr/created_and_deleted_shard1_replica1/ shard1
   [junit4]   2> 905217 INFO  
(zkCallback-526-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/created_and_deleted/state.json] for collection 
[created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 905268 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 905269 INFO  (qtp1388419579-3870) [n:127.0.0.1:51242_solr 
c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=created_and_deleted_shard1_replica1&action=CREATE&numShards=1&collection=created_and_deleted&shard=shard1&wt=javabin&version=2}
 status=0 QTime=3672
   [junit4]   2> 905271 INFO  (qtp1243735339-3833) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 905336 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 905372 INFO  
(zkCallback-526-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/created_and_deleted/state.json] for collection 
[created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 906272 INFO  (qtp1243735339-3833) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=1&collection.configName=conf&name=created_and_deleted&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=4939
   [junit4]   2> 906274 INFO  (qtp1243735339-3834) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 906275 INFO  (qtp1243735339-3834) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 906277 INFO  (qtp1243735339-3832) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params 
name=created_and_deleted&action=DELETE&wt=javabin&version=2 and 
sendToOCPQueue=true
   [junit4]   2> 906279 INFO  
(OverseerThreadFactory-1966-thread-5-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing 
Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 906281 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: 
solr.core.created_and_deleted.shard1.replica1
   [junit4]   2> 906282 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.c.SolrCore [created_and_deleted_shard1_replica1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@15715f6c
   [junit4]   2> 906291 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: 
solr.core.created_and_deleted.shard1.replica1
   [junit4]   2> 906292 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.s.h.HdfsDirectory Closing hdfs directory 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data/snapshot_metadata
   [junit4]   2> 906299 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.s.h.HdfsDirectory Closing hdfs directory 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data/index
   [junit4]   2> 906299 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.s.h.HdfsDirectory Closing hdfs directory 
hdfs://localhost:35454/data/created_and_deleted/core_node1/data
   [junit4]   2> 906324 INFO  (IPC Server handler 9 on 35454) [    ] 
BlockStateChange BLOCK* addToInvalidates: blk_1073741825_1001 127.0.0.1:50821 
127.0.0.1:32824 
   [junit4]   2> 906334 INFO  (qtp1388419579-3871) [n:127.0.0.1:51242_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={deleteInstanceDir=true&core=created_and_deleted_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2}
 status=0 QTime=53
   [junit4]   2> 907038 INFO  (qtp1243735339-3832) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={name=created_and_deleted&action=DELETE&wt=javabin&version=2} status=0 
QTime=760
   [junit4]   2> 907039 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 907040 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params 
action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 907040 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 907041 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testCreationAndDeletion
   [junit4]   2> 907089 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testTooManyReplicas
   [junit4]   2> 907090 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 907093 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 907095 INFO  
(OverseerThreadFactory-1966-thread-5-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
collection
   [junit4]   2> 907095 WARN  
(OverseerThreadFactory-1966-thread-5-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Specified 
replicationFactor of 10 on collection collection is higher than or equal to the 
number of Solr instances currently live or live and part of your 
createNodeSet(4). It's unusual to run two replica of the same slice on the same 
Solr-instance.
   [junit4]   2> 907095 ERROR 
(OverseerThreadFactory-1966-thread-5-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerCollectionMessageHandler 
Collection: collection operation: create 
failed:org.apache.solr.common.SolrException: Cannot create collection 
collection. Value of maxShardsPerNode is 1, and the number of nodes currently 
live or live and part of your createNodeSet is 4. This allows a maximum of 4 to 
be created. Value of numShards is 2 and value of replicationFactor is 10. This 
requires 20 shards to be created (higher than the allowed number)
   [junit4]   2>        at 
org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
   [junit4]   2>        at 
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 907097 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2}
 status=400 QTime=3
   [junit4]   2> 907098 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testTooManyReplicas
   [junit4]   2> 907152 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.addReplicaTest-seed#[5A047B097F177FD2])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting addReplicaTest
   [junit4]   2> 907153 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.addReplicaTest-seed#[5A047B097F177FD2])
 [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 907155 INFO  (qtp1243735339-3829) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=4&name=addReplicaColl&action=CREATE&numShards=2&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 907156 INFO  
(OverseerThreadFactory-1966-thread-5-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
addReplicaColl
   [junit4]   2> 907157 INFO  
(OverseerCollectionConfigSetProcessor-97551327629934602-127.0.0.1:51242_solr-n_0000000000)
 [n:127.0.0.1:51242_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000010 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 907160 INFO  
(org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@61e526d2)
 [    ] BlockStateChange BLOCK* BlockManager: ask 127.0.0.1:50821 to delete 
[blk_1073741825_1001]
   [junit4]   2> 907370 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
   [junit4]   2> 907374 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
   [junit4]   2> 907420 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
   [junit4]   2> 907432 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
   [junit4]   2> 907535 INFO  
(zkCallback-523-thread-1-processing-n:127.0.0.1:46765_solr) 
[n:127.0.0.1:46765_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] 
has occurred - updating... (live nodes size: [4])
   [junit4]   2> 907535 INFO  
(zkCallback-526-thread-2-processing-n:127.0.0.1:51242_solr) 
[n:127.0.0.1:51242_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] 
has occurred - updating... (live nodes size: [4])
   [junit4]   2> 907535 INFO  
(zkCallback-525-thread-1-processing-n:127.0.0.1:54331_solr) 
[n:127.0.0.1:54331_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] 
has occurred - updating... (live nodes size: [4])
   [junit4]   2> 907535 INFO  
(zkCallback-527-thread-2-processing-n:127.0.0.1:58634_solr) 
[n:127.0.0.1:58634_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] 
has occurred - updating... (live nodes size: [4])
   [junit4]   2> 908378 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.2
   [junit4]   2> 908403 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.s.IndexSchema [addReplicaColl_shard2_replica2] Schema name=minimal
   [junit4]   2> 908405 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 908406 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.2
   [junit4]   2> 908406 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica2' using 
configuration from collection addReplicaColl
   [junit4]   2> 908406 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35454/data
   [junit4]   2> 908406 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 908407 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] o.a.s.c.SolrCore 
[[addReplicaColl_shard2_replica2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node3/addReplicaColl_shard2_replica2],
 dataDir=[hdfs://localhost:35454/data/addReplicaColl/core_node1/data/]
   [junit4]   2> 908407 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node1/data/snapshot_metadata
   [junit4]   2> 908414 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.s.IndexSchema [addReplicaColl_shard1_replica2] Schema name=minimal
   [junit4]   2> 908415 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908415 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908415 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908416 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 908417 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica2' using 
configuration from collection addReplicaColl
   [junit4]   2> 908418 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35454/data
   [junit4]   2> 908419 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 908419 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] o.a.s.c.SolrCore 
[[addReplicaColl_shard1_replica2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node2/addReplicaColl_shard1_replica2],
 dataDir=[hdfs://localhost:35454/data/addReplicaColl/core_node2/data/]
   [junit4]   2> 908419 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node2/data/snapshot_metadata
   [junit4]   2> 908426 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908426 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908426 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908428 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.2
   [junit4]   2> 908448 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.2
   [junit4]   2> 908451 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.s.IndexSchema [addReplicaColl_shard2_replica1] Schema name=minimal
   [junit4]   2> 908453 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 908453 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.s.IndexSchema [addReplicaColl_shard1_replica1] Schema name=minimal
   [junit4]   2> 908454 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica1' using 
configuration from collection addReplicaColl
   [junit4]   2> 908454 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35454/data
   [junit4]   2> 908454 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 908454 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] o.a.s.c.SolrCore 
[[addReplicaColl_shard2_replica1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node4/addReplicaColl_shard2_replica1],
 dataDir=[hdfs://localhost:35454/data/addReplicaColl/core_node3/data/]
   [junit4]   2> 908455 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node3/data/snapshot_metadata
   [junit4]   2> 908455 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 908456 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica1' using 
configuration from collection addReplicaColl
   [junit4]   2> 908456 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35454/data
   [junit4]   2> 908456 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 908456 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] o.a.s.c.SolrCore 
[[addReplicaColl_shard1_replica1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_5A047B097F177FD2-001/tempDir-001/node1/addReplicaColl_shard1_replica1],
 dataDir=[hdfs://localhost:35454/data/addReplicaColl/core_node4/data/]
   [junit4]   2> 908456 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node4/data/snapshot_metadata
   [junit4]   2> 908462 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908462 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908462 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908469 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 908470 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node2/data
   [junit4]   2> 908470 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 908470 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908470 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908470 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908477 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node1/data
   [junit4]   2> 908486 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 908487 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node4/data
   [junit4]   2> 908488 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 908489 INFO  (qtp739461459-3852) [n:127.0.0.1:46765_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node3/data
   [junit4]   2> 908502 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node2/data/index
   [junit4]   2> 908505 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node1/data/index
   [junit4]   2> 908507 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908507 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908507 INFO  (qtp1243735339-3835) [n:127.0.0.1:58634_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908542 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908542 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908542 INFO  (qtp1388419579-3865) [n:127.0.0.1:51242_solr 
c:addReplicaColl s:shard2  x:addReplicaColl_shard2_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
   [junit4]   2> 908544 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:35454/data/addReplicaColl/core_node4/data/index
   [junit4]   2> 908549 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 908549 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addReplicaColl_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 908549 INFO  (qtp1134897283-3844) [n:127.0.0.1:54331_solr 
c:addReplicaColl s:shard1  x:addRe

[...truncated too long message...]

erverCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 4409159 INFO  
(zkCallback-3650-thread-4-processing-n:127.0.0.1:49093_solr) 
[n:127.0.0.1:49093_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (2) -> (1)
   [junit4]   2> 4409165 INFO  (jetty-launcher-3627-thread-17) [    ] 
o.a.s.c.Overseer Overseer 
(id=97551504515792910-127.0.0.1:49093_solr-n_0000000006) closing
   [junit4]   2> 4409170 INFO  
(OverseerStateUpdate-97551504515792910-127.0.0.1:49093_solr-n_0000000006) 
[n:127.0.0.1:49093_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:49093_solr
   [junit4]   2> 4410512 WARN  
(zkCallback-3671-thread-4-processing-n:127.0.0.1:37351_solr) 
[n:127.0.0.1:37351_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 4410512 INFO  (jetty-launcher-3627-thread-14) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 4410512 INFO  (jetty-launcher-3627-thread-14) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@5a7d3aaa{/solr,null,UNAVAILABLE}
   [junit4]   2> 4410645 WARN  
(zkCallback-3670-thread-4-processing-n:127.0.0.1:38343_solr) 
[n:127.0.0.1:38343_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 4410646 INFO  (jetty-launcher-3627-thread-12) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 4410646 INFO  (jetty-launcher-3627-thread-12) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1cb442ec{/solr,null,UNAVAILABLE}
   [junit4]   2> 4410654 WARN  
(zkCallback-3664-thread-4-processing-n:127.0.0.1:49575_solr) 
[n:127.0.0.1:49575_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 4410654 INFO  (jetty-launcher-3627-thread-15) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 4410654 INFO  (jetty-launcher-3627-thread-15) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@576677f7{/solr,null,UNAVAILABLE}
   [junit4]   2> 4410659 WARN  
(zkCallback-3685-thread-6-processing-n:127.0.0.1:43236_solr) 
[n:127.0.0.1:43236_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 4410659 INFO  (jetty-launcher-3627-thread-11) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 4410659 INFO  (jetty-launcher-3627-thread-11) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@792ee9e7{/solr,null,UNAVAILABLE}
   [junit4]   2> 4410666 WARN  
(zkCallback-3650-thread-5-processing-n:127.0.0.1:49093_solr) 
[n:127.0.0.1:49093_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, 
but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 4410666 INFO  (jetty-launcher-3627-thread-17) [    ] 
o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 4410666 INFO  (jetty-launcher-3627-thread-17) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@7279e548{/solr,null,UNAVAILABLE}
   [junit4]   2> 4410667 INFO  
(SUITE-TestStressCloudBlindAtomicUpdates-seed#[5A047B097F177FD2]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:50588 50588
   [junit4]   2> 4410769 INFO  (Thread-60459) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:50588 50588
   [junit4]   2> 4410770 WARN  (Thread-60459) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/aliases.json
   [junit4]   2>        11      /solr/clusterprops.json
   [junit4]   2>        10      
/solr/configs/org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates_config-set
   [junit4]   2>        10      /solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/clusterstate.json
   [junit4]   2>        10      /solr/collections/test_col/state.json
   [junit4]   2>        3       
/solr/overseer_elect/election/97551504515792917-127.0.0.1:49575_solr-n_0000000001
   [junit4]   2>        3       
/solr/overseer_elect/election/97551504515792910-127.0.0.1:49093_solr-n_0000000006
   [junit4]   2>        2       
/solr/overseer_elect/election/97551504515792920-127.0.0.1:38343_solr-n_0000000004
   [junit4]   2>        2       
/solr/overseer_elect/election/97551504515792916-127.0.0.1:54205_solr-n_0000000002
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        811     /solr/overseer/collection-queue-work
   [junit4]   2>        81      /solr/overseer/queue
   [junit4]   2>        11      /solr/live_nodes
   [junit4]   2>        11      /solr/collections
   [junit4]   2>        9       /solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 4410770 INFO  
(SUITE-TestStressCloudBlindAtomicUpdates-seed#[5A047B097F177FD2]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.4/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestStressCloudBlindAtomicUpdates_5A047B097F177FD2-001
   [junit4]   2> Mar 03, 2017 5:24:06 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: 
codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY,
 chunkSize=5, maxDocsPerChunk=1, blockSize=9), 
termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, 
chunkSize=5, blockSize=9)), sim=RandomSimilarity(queryNorm=false,coord=yes): 
{}, locale=tr-TR, timezone=America/Nome
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_121 (64-bit)/cpus=4,threads=1,free=44745832,total=533725184
   [junit4]   2> NOTE: All tests run in this JVM: [TestCloudPseudoReturnFields, 
RemoteQueryErrorTest, MetricsHandlerTest, PrimUtilsTest, SolrTestCaseJ4Test, 
TestRuleBasedAuthorizationPlugin, DirectUpdateHandlerOptimizeTest, 
DistributedFacetExistsSmallTest, TestUtils, TestDefaultStatsCache, 
TestDynamicFieldCollectionResource, LeaderInitiatedRecoveryOnShardRestartTest, 
TestManagedSchema, SolrCoreCheckLockOnStartupTest, SpatialHeatmapFacetsTest, 
GraphQueryTest, UtilsToolTest, TestOverriddenPrefixQueryForCustomFieldType, 
DirectUpdateHandlerTest, NoCacheHeaderTest, TestConfigSetsAPIZkFailure, 
TestManagedSynonymFilterFactory, TestJsonRequest, TestIndexingPerformance, 
TestShortCircuitedRequests, TestRequestStatusCollectionAPI, 
ConnectionReuseTest, SuggesterWFSTTest, TestGeoJSONResponseWriter, 
CurrencyFieldXmlFileTest, TestSolrDeletionPolicy1, PrimitiveFieldTypeTest, 
TestCollationField, TestShardHandlerFactory, TestSolrQueryParser, 
ResponseHeaderTest, FacetPivotSmallTest, TestConfig, 
TestClassicSimilarityFactory, TestStressReorder, AddBlockUpdateTest, 
HdfsDirectoryFactoryTest, TemplateUpdateProcessorTest, TestExactStatsCache, 
TestWriterPerf, TestLFUCache, DistributedFacetPivotSmallAdvancedTest, 
TestReloadDeadlock, TestFieldCacheVsDocValues, ConnectionManagerTest, 
ChangedSchemaMergeTest, TestSort, TestMiniSolrCloudCluster, 
OpenExchangeRatesOrgProviderTest, UUIDUpdateProcessorFallbackTest, 
BinaryUpdateRequestHandlerTest, CursorMarkTest, BitVectorTest, 
TestHighFrequencyDictionaryFactory, QueryParsingTest, BadIndexSchemaTest, 
CdcrBootstrapTest, OverseerTaskQueueTest, TestUnifiedSolrHighlighter, 
TestSolrQueryParserResource, PluginInfoTest, DeleteInactiveReplicaTest, 
HdfsChaosMonkeySafeLeaderTest, TestSolrConfigHandlerCloud, 
CollectionStateFormat2Test, TestConfigOverlay, LeaderElectionTest, 
SortSpecParsingTest, DebugComponentTest, DataDrivenBlockJoinTest, 
SolrJmxReporterTest, DOMUtilTest, TestIndexSearcher, TestJmxMonitoredMap, 
TestSolrXml, DistribCursorPagingTest, CleanupOldIndexTest, 
ParsingFieldUpdateProcessorsTest, TestDocTermOrds, TestSolrQueryResponse, 
SolrMetricsIntegrationTest, TestRebalanceLeaders, SuggesterTest, 
TestSchemaVersionResource, BadComponentTest, SaslZkACLProviderTest, 
SpellPossibilityIteratorTest, TestTolerantSearch, 
TestLeaderInitiatedRecoveryThread, AssignTest, TestPushWriter, TestRTimerTree, 
TestGroupingSearch, TlogReplayBufferedWhileIndexingTest, TestXIncludeConfig, 
TestMaxScoreQueryParser, UpdateParamsTest, TestDistributedMissingSort, 
TestOnReconnectListenerSupport, TimeZoneUtilsTest, SolrPluginUtilsTest, 
LukeRequestHandlerTest, TestUseDocValuesAsStored2, 
OverseerModifyCollectionTest, TestRecovery, 
TestDistributedStatsComponentCardinality, SolrCloudExampleTest, 
BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, 
OpenCloseCoreStressTest, TestZkChroot, ShardRoutingCustomTest, 
TestDistributedSearch, CdcrVersionReplicationTest, 
ConcurrentDeleteAndCreateCollectionTest, DeleteNodeTest, DeleteShardTest, 
DistribDocExpirationUpdateProcessorTest, HttpPartitionTest, 
LeaderFailureAfterFreshStartTest, RecoveryAfterSoftCommitTest, 
RollingRestartTest, SharedFSAutoReplicaFailoverUtilsTest, 
TestAuthenticationFramework, TestCloudDeleteByQuery, TestClusterProperties, 
TestCollectionAPI, TestConfigSetsAPIExclusivity, TestCryptoKeys, 
TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, 
TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, 
TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, 
TestLeaderElectionZkExpiry, TestRandomFlRTGCloud, TestSSLRandomization, 
TestSizeLimitedDistributedMap, TestSolrCloudWithDelegationTokens, 
TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, 
TestStressCloudBlindAtomicUpdates]
   [junit4] Completed [536/680 (3!)] on J2 in 817.01s, 5 tests, 1 failure <<< 
FAILURES!

[...truncated 62861 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to