Build: https://builds.apache.org/job/Lucene-Solr-Tests-6.x/737/

1 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; 
[core_node3:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:49782","node_name":"127.0.0.1:49782_","state":"active","leader":"true"}];
 clusterState: 
DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={   
"replicationFactor":"3",   "shards":{"shard1":{       
"range":"80000000-7fffffff",       "state":"active",       "replicas":{         
"core_node1":{           "core":"c8n_1x3_lf_shard1_replica2",           
"base_url":"http://127.0.0.1:37680";,           "node_name":"127.0.0.1:37680_",  
         "state":"down"},         "core_node2":{           "state":"down",      
     "base_url":"http://127.0.0.1:34191";,           
"core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:34191_"}, 
        "core_node3":{           "core":"c8n_1x3_lf_shard1_replica3",           
"base_url":"http://127.0.0.1:49782";,           "node_name":"127.0.0.1:49782_",  
         "state":"active",           "leader":"true"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"1",   
"autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 
1; 
[core_node3:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:49782","node_name":"127.0.0.1:49782_","state":"active","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:37680";,
          "node_name":"127.0.0.1:37680_",
          "state":"down"},
        "core_node2":{
          "state":"down",
          "base_url":"http://127.0.0.1:34191";,
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:34191_"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:49782";,
          "node_name":"127.0.0.1:49782_",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
        at 
__randomizedtesting.SeedInfo.seed([3481232CACD4C33C:BCD51CF60228AEC4]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:168)
        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:55)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12508 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/init-core-data-001
   [junit4]   2> 1520545 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[3481232CACD4C33C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 1520546 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[3481232CACD4C33C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 1520546 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[3481232CACD4C33C]-worker) [    ] 
o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1520548 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1520548 INFO  (Thread-2469) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1520548 INFO  (Thread-2469) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 1520648 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkTestServer start zk server on port:58000
   [junit4]   2> 1520655 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
 to /configs/conf1/solrconfig.xml
   [junit4]   2> 1520656 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
 to /configs/conf1/schema.xml
   [junit4]   2> 1520657 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
 to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1520658 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
 to /configs/conf1/stopwords.txt
   [junit4]   2> 1520659 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt
 to /configs/conf1/protwords.txt
   [junit4]   2> 1520660 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/currency.xml
 to /configs/conf1/currency.xml
   [junit4]   2> 1520661 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
 to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1520662 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
 to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1520663 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
 to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1520664 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
 to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1520665 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractZkTestCase put 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
 to /configs/conf1/synonyms.txt
   [junit4]   2> 1520731 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/cores/collection1
   [junit4]   2> 1520733 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1520734 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@633a38c8{/,null,AVAILABLE}
   [junit4]   2> 1520735 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@13ed3701{HTTP/1.1,[http/1.1]}{127.0.0.1:46890}
   [junit4]   2> 1520735 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server Started @1523957ms
   [junit4]   2> 1520736 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/tempDir-001/control/data,
 hostContext=/, hostPort=34191, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/cores}
   [junit4]   2> 1520736 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1520736 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.5.0
   [junit4]   2> 1520736 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1520736 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1520737 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-02-19T21:55:10.568Z
   [junit4]   2> 1520739 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1520739 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/solr.xml
   [junit4]   2> 1520752 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1520752 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58000/solr
   [junit4]   2> 1520765 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1520765 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.OverseerElectionContext I am going to be the 
leader 127.0.0.1:34191_
   [junit4]   2> 1520766 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.Overseer Overseer 
(id=97487507317063684-127.0.0.1:34191_-n_0000000000) starting
   [junit4]   2> 1520770 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:34191_
   [junit4]   2> 1520772 INFO  
(zkCallback-3715-thread-1-processing-n:127.0.0.1:34191_) [n:127.0.0.1:34191_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1520824 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions 
underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/cores
   [junit4]   2> 1520824 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:34191_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1520828 INFO  
(OverseerStateUpdate-97487507317063684-127.0.0.1:34191_-n_0000000000) 
[n:127.0.0.1:34191_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 1521837 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 1521847 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1521943 WARN  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1521945 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.s.IndexSchema 
Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1521960 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection 
control_collection
   [junit4]   2> 1521965 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/control-001/cores/collection1/data/]
   [junit4]   2> 1521966 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX 
Server: com.sun.jmx.mbeanserver.JmxMBeanServer@60b9a978
   [junit4]   2> 1521971 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: 
minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.2949087470865671]
   [junit4]   2> 1521977 WARN  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = 
requestHandler,name = /dump,class = DumpRequestHandler,attributes = 
{initParams=a, name=/dump, class=DumpRequestHandler},args = 
{defaults={a=A,b=B}}}
   [junit4]   2> 1522002 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1522002 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1522002 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1522002 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1522003 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=47, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=23.5439453125, 
floorSegmentMB=1.419921875, forceMergeDeletesPctAllowed=15.213861280101709, 
segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.37979800645846007
   [junit4]   2> 1522003 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.s.SolrIndexSearcher Opening [Searcher@7556c6d1[collection1] main]
   [junit4]   2> 1522004 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1522004 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1522004 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1522004 INFO  
(coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_ c:control_collection   x:collection1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1559800118598696960
   [junit4]   2> 1522007 INFO  
(searcherExecutor-6484-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@7556c6d1[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1522013 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas 
found to continue.
   [junit4]   2> 1522014 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new 
leader - try and sync
   [junit4]   2> 1522014 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:34191/collection1/
   [junit4]   2> 1522014 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync 
replicas to me
   [junit4]   2> 1522014 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.SyncStrategy 
http://127.0.0.1:34191/collection1/ has no replicas
   [junit4]   2> 1522016 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new 
leader: http://127.0.0.1:34191/collection1/ shard1
   [junit4]   2> 1522166 INFO  
(coreZkRegister-6476-thread-1-processing-n:127.0.0.1:34191_ x:collection1 
c:control_collection) [n:127.0.0.1:34191_ c:control_collection s:shard1 
r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery 
necessary
   [junit4]   2> 1522381 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1522383 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58000/solr ready
   [junit4]   2> 1522383 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection 
loss:false
   [junit4]   2> 1522383 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 1522478 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/cores/collection1
   [junit4]   2> 1522478 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001
   [junit4]   2> 1522479 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@3d045f71{/,null,AVAILABLE}
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@631b0a6a{HTTP/1.1,[http/1.1]}{127.0.0.1:33593}
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server Started @1525750ms
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/tempDir-001/jetty1,
 solrconfig=solrconfig.xml, hostContext=/, hostPort=37680, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/cores}
   [junit4]   2> 1522528 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.5.0
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1522528 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1522529 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-02-19T21:55:12.360Z
   [junit4]   2> 1522556 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1522557 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/solr.xml
   [junit4]   2> 1522561 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1522580 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58000/solr
   [junit4]   2> 1522592 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:37680_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1522593 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:37680_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1522595 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:37680_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:37680_
   [junit4]   2> 1522596 INFO  (zkCallback-3719-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1522596 INFO  
(zkCallback-3715-thread-3-processing-n:127.0.0.1:34191_) [n:127.0.0.1:34191_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1522609 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1522697 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:37680_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions 
underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/cores
   [junit4]   2> 1522697 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:37680_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1522703 INFO  
(OverseerStateUpdate-97487507317063684-127.0.0.1:34191_-n_0000000000) 
[n:127.0.0.1:34191_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard2
   [junit4]   2> 1522805 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1523721 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.5.0
   [junit4]   2> 1523731 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1523798 WARN  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1523800 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded 
schema test/1.0 with uniqueid field id
   [junit4]   2> 1523806 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1523806 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 1523807 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX 
monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@60b9a978
   [junit4]   2> 1523807 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: 
[LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, 
maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.2949087470865671]
   [junit4]   2> 1523811 WARN  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.c.RequestHandlers 
INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class 
= DumpRequestHandler,attributes = {initParams=a, name=/dump, 
class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1523835 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using 
UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1523835 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1523835 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard 
AutoCommit: disabled
   [junit4]   2> 1523835 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft 
AutoCommit: disabled
   [junit4]   2> 1523836 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: 
[TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=27, 
maxMergedSegmentMB=23.5439453125, floorSegmentMB=1.419921875, 
forceMergeDeletesPctAllowed=15.213861280101709, segmentsPerTier=50.0, 
maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37979800645846007
   [junit4]   2> 1523836 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@75310dfa[collection1] main]
   [junit4]   2> 1523837 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1523837 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1523837 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler 
Commits will be reserved for  10000
   [junit4]   2> 1523838 INFO  
(searcherExecutor-6495-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@75310dfa[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1523838 INFO  
(coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:37680_) 
[n:127.0.0.1:37680_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not 
find max version in index or recent updates, using new clock 1559800120521785344
   [junit4]   2> 1523843 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to 
continue.
   [junit4]   2> 1523843 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try 
and sync
   [junit4]   2> 1523843 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:37680/collection1/
   [junit4]   2> 1523843 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1523843 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:37680/collection1/ has no 
replicas
   [junit4]   2> 1523846 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:37680/collection1/ shard2
   [junit4]   2> 1523948 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1523999 INFO  
(coreZkRegister-6489-thread-1-processing-n:127.0.0.1:37680_ x:collection1 
c:collection1) [n:127.0.0.1:37680_ c:collection1 s:shard2 r:core_node1 
x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1524108 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1524329 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/cores/collection1
   [junit4]   2> 1524330 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001
   [junit4]   2> 1524330 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1524331 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@4b12a83c{/,null,AVAILABLE}
   [junit4]   2> 1524332 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@44f1f5ec{HTTP/1.1,[http/1.1]}{127.0.0.1:53877}
   [junit4]   2> 1524332 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server Started @1527554ms
   [junit4]   2> 1524332 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/tempDir-001/jetty2,
 solrconfig=solrconfig.xml, hostContext=/, hostPort=49782, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/cores}
   [junit4]   2> 1524332 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1524333 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.5.0
   [junit4]   2> 1524333 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1524333 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1524333 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-02-19T21:55:14.164Z
   [junit4]   2> 1524335 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1524335 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/solr.xml
   [junit4]   2> 1524340 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1524340 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58000/solr
   [junit4]   2> 1524348 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:49782_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (2)
   [junit4]   2> 1524350 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:49782_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1524351 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:49782_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:49782_
   [junit4]   2> 1524352 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1524352 INFO  (zkCallback-3719-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1524355 INFO  
(zkCallback-3715-thread-1-processing-n:127.0.0.1:34191_) [n:127.0.0.1:34191_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1524355 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1524413 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:49782_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions 
underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/cores
   [junit4]   2> 1524414 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:49782_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1524419 INFO  
(OverseerStateUpdate-97487507317063684-127.0.0.1:34191_-n_0000000000) 
[n:127.0.0.1:34191_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard1
   [junit4]   2> 1524522 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1524522 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1525438 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.5.0
   [junit4]   2> 1525457 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1525646 WARN  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1525648 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded 
schema test/1.0 with uniqueid field id
   [junit4]   2> 1525662 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1525663 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 1525663 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX 
monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@60b9a978
   [junit4]   2> 1525672 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: 
[LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, 
maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.2949087470865671]
   [junit4]   2> 1525676 WARN  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.c.RequestHandlers 
INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class 
= DumpRequestHandler,attributes = {initParams=a, name=/dump, 
class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1525723 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using 
UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1525723 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1525724 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard 
AutoCommit: disabled
   [junit4]   2> 1525724 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft 
AutoCommit: disabled
   [junit4]   2> 1525724 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: 
[TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=27, 
maxMergedSegmentMB=23.5439453125, floorSegmentMB=1.419921875, 
forceMergeDeletesPctAllowed=15.213861280101709, segmentsPerTier=50.0, 
maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37979800645846007
   [junit4]   2> 1525724 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@2c0c6250[collection1] main]
   [junit4]   2> 1525725 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1525725 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1525725 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler 
Commits will be reserved for  10000
   [junit4]   2> 1525726 INFO  
(coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:49782_) 
[n:127.0.0.1:49782_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not 
find max version in index or recent updates, using new clock 1559800122501496832
   [junit4]   2> 1525744 INFO  
(searcherExecutor-6506-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@2c0c6250[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1525746 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to 
continue.
   [junit4]   2> 1525746 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try 
and sync
   [junit4]   2> 1525746 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy Sync replicas to 
http://127.0.0.1:49782/collection1/
   [junit4]   2> 1525746 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1525746 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:49782/collection1/ has no 
replicas
   [junit4]   2> 1525748 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:49782/collection1/ shard1
   [junit4]   2> 1525849 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1525849 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1525899 INFO  
(coreZkRegister-6500-thread-1-processing-n:127.0.0.1:49782_ x:collection1 
c:collection1) [n:127.0.0.1:49782_ c:collection1 s:shard1 r:core_node2 
x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1526002 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1526002 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [3])
   [junit4]   2> 1526121 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/cores/collection1
   [junit4]   2> 1526121 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001
   [junit4]   2> 1526122 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1526160 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@5d3369f6{/,null,AVAILABLE}
   [junit4]   2> 1526160 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@167693aa{HTTP/1.1,[http/1.1]}{127.0.0.1:49107}
   [junit4]   2> 1526160 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.e.j.s.Server Started @1529382ms
   [junit4]   2> 1526160 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/tempDir-001/jetty3,
 solrconfig=solrconfig.xml, hostContext=/, hostPort=52907, 
coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/cores}
   [junit4]   2> 1526161 ERROR 
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1526161 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.5.0
   [junit4]   2> 1526161 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1526161 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1526161 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2017-02-19T21:55:15.992Z
   [junit4]   2> 1526172 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in 
ZooKeeper)
   [junit4]   2> 1526172 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/solr.xml
   [junit4]   2> 1526179 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1526179 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58000/solr
   [junit4]   2> 1526218 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:52907_    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (3)
   [junit4]   2> 1526225 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:52907_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1526231 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:52907_    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:52907_
   [junit4]   2> 1526232 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1526232 INFO  (zkCallback-3719-thread-1) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1526232 INFO  
(zkCallback-3715-thread-1-processing-n:127.0.0.1:34191_) [n:127.0.0.1:34191_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1526232 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1526232 INFO  
(zkCallback-3737-thread-1-processing-n:127.0.0.1:52907_) [n:127.0.0.1:52907_    
] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1526318 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:52907_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions 
underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/cores
   [junit4]   2> 1526318 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) 
[n:127.0.0.1:52907_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1526320 INFO  
(OverseerStateUpdate-97487507317063684-127.0.0.1:34191_-n_0000000000) 
[n:127.0.0.1:34191_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard 
shard=shard2
   [junit4]   2> 1526428 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1526430 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1526430 INFO  
(zkCallback-3737-thread-1-processing-n:127.0.0.1:52907_) [n:127.0.0.1:52907_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1527338 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using 
Lucene MatchVersion: 6.5.0
   [junit4]   2> 1527348 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] Schema name=test
   [junit4]   2> 1527418 WARN  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.s.IndexSchema 
[collection1] default search field in schema is text. WARNING: Deprecated, 
please use 'df' on request instead.
   [junit4]   2> 1527420 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded 
schema test/1.0 with uniqueid field id
   [junit4]   2> 1527434 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.c.CoreContainer 
Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1527435 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.c.SolrCore 
[[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/cores/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 1527435 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX 
monitoring is enabled. Adding Solr mbeans to JMX Server: 
com.sun.jmx.mbeanserver.JmxMBeanServer@60b9a978
   [junit4]   2> 1527436 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: 
[LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, 
maxMergeSize=9223372036854775807, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.2949087470865671]
   [junit4]   2> 1527439 WARN  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.c.RequestHandlers 
INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class 
= DumpRequestHandler,attributes = {initParams=a, name=/dump, 
class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1527464 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using 
UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1527464 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 
maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1527465 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard 
AutoCommit: disabled
   [junit4]   2> 1527465 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft 
AutoCommit: disabled
   [junit4]   2> 1527465 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy 
RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: 
[TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=27, 
maxMergedSegmentMB=23.5439453125, floorSegmentMB=1.419921875, 
forceMergeDeletesPctAllowed=15.213861280101709, segmentsPerTier=50.0, 
maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37979800645846007
   [junit4]   2> 1527466 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher 
Opening [Searcher@1d7e5ea5[collection1] main]
   [junit4]   2> 1527466 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf1
   [junit4]   2> 1527466 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1527466 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler 
Commits will be reserved for  10000
   [junit4]   2> 1527467 INFO  
(coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52907_) 
[n:127.0.0.1:52907_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not 
find max version in index or recent updates, using new clock 1559800124327067648
   [junit4]   2> 1527470 INFO  
(searcherExecutor-6517-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
c:collection1) [n:127.0.0.1:52907_ c:collection1   x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@1d7e5ea5[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1527471 INFO  
(coreZkRegister-6511-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
c:collection1) [n:127.0.0.1:52907_ c:collection1 s:shard2 r:core_node3 
x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 1527471 INFO  
(updateExecutor-3734-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1527471 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. 
recoveringAfterStartup=true
   [junit4]   2> 1527471 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 1527471 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. 
core=[collection1]
   [junit4]   2> 1527472 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. 
FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 1527472 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core 
[collection1] as recovering, leader is [http://127.0.0.1:37680/collection1/] 
and I am [http://127.0.0.1:52907/collection1/]
   [junit4]   2> 1527474 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery 
command to [http://127.0.0.1:37680]; [WaitForState: 
action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52907_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 1527475 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40019,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1527495 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53137], receiveBufferSize=530904
   [junit4]   2> 1527501 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: 
recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 1527501 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 
(shard2 of collection1) have state: recovering
   [junit4]   2> 1527501 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, 
shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? 
true, live=true, checkLive=true, currentState=down, localState=active, 
nodeName=127.0.0.1:52907_, coreNodeName=core_node3, 
onlyIfActiveCheckResult=false, nodeProps: 
core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52907","node_name":"127.0.0.1:52907_","state":"down"}
   [junit4]   2> 1527574 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1527574 INFO  
(zkCallback-3737-thread-1-processing-n:127.0.0.1:52907_) [n:127.0.0.1:52907_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1527574 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1527919 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1527919 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 
30000 for each attempt
   [junit4]   2> 1527919 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: 
collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 1528502 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, 
shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? 
true, live=true, checkLive=true, currentState=recovering, localState=active, 
nodeName=127.0.0.1:52907_, coreNodeName=core_node3, 
onlyIfActiveCheckResult=false, nodeProps: 
core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52907","node_name":"127.0.0.1:52907_","state":"recovering"}
   [junit4]   2> 1528502 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, 
checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 1528502 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores 
params={nodeName=127.0.0.1:52907_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2}
 status=0 QTime=1000
   [junit4]   2> 1535503 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync 
from [http://127.0.0.1:37680/collection1/] - recoveringAfterStartup=[true]
   [junit4]   2> 1535503 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 
url=http://127.0.0.1:52907 START replicas=[http://127.0.0.1:37680/collection1/] 
nUpdates=100
   [junit4]   2> 1535506 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40062,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1535506 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53180], receiveBufferSize=530904
   [junit4]   2> 1535508 INFO  (qtp950265001-17902) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint 
IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, 
maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, 
maxDoc=0}
   [junit4]   2> 1535508 INFO  (qtp950265001-17902) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request 
[collection1]  webapp= path=/get 
params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 1535509 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint 
millis:0.0 result:{maxVersionSpecified=9223372036854775807, 
maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, 
maxDoc=0}
   [junit4]   2> 1535509 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to 
do a PeerSync 
   [junit4]   2> 1535509 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1535509 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted 
changes. Skipping IW.commit.
   [junit4]   2> 1535510 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1535510 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery 
was successful.
   [junit4]   2> 1535510 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered 
during PeerSync.
   [junit4]   2> 1535510 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
   [junit4]   2> 1535510 INFO  
(recoveryExecutor-3735-thread-1-processing-n:127.0.0.1:52907_ x:collection1 
s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52907_ c:collection1 s:shard2 
r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active 
after recovery.
   [junit4]   2> 1535513 INFO  
(zkCallback-3737-thread-1-processing-n:127.0.0.1:52907_) [n:127.0.0.1:52907_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1535513 INFO  
(zkCallback-3731-thread-1-processing-n:127.0.0.1:49782_) [n:127.0.0.1:49782_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1535514 INFO  
(zkCallback-3725-thread-1-processing-n:127.0.0.1:37680_) [n:127.0.0.1:37680_    
] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent 
state:SyncConnected type:NodeDataChanged 
path:/collections/collection1/state.json] for collection [collection1] has 
occurred - updating... (live nodes size: [4])
   [junit4]   2> 1535923 INFO  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 1535924 INFO  (SocketProxy-Acceptor-34191) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=38811,localport=34191], receiveBufferSize:531000
   [junit4]   2> 1535925 INFO  (SocketProxy-Acceptor-34191) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=46890,localport=50845], receiveBufferSize=530904
   [junit4]   2> 1535928 INFO  (qtp1841075260-17857) [n:127.0.0.1:34191_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.DirectUpdateHandler2 start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1535928 INFO  (qtp1841075260-17857) [n:127.0.0.1:34191_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1535929 INFO  (qtp1841075260-17857) [n:127.0.0.1:34191_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1535929 INFO  (qtp1841075260-17857) [n:127.0.0.1:34191_ 
c:control_collection s:shard1 r:core_node1 x:collection1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update 
params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
 0 1
   [junit4]   2> 1535930 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40072,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1535931 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53190], receiveBufferSize=530904
   [junit4]   2> 1535933 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=35666,localport=49782], receiveBufferSize:531000
   [junit4]   2> 1535933 INFO  (SocketProxy-Acceptor-52907) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=36964,localport=52907], receiveBufferSize:531000
   [junit4]   2> 1535939 INFO  (qtp861531069-17929) [n:127.0.0.1:49782_ 
c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 
start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1535939 INFO  (qtp861531069-17929) [n:127.0.0.1:49782_ 
c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 
No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1535943 INFO  (SocketProxy-Acceptor-52907) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=49107,localport=51939], receiveBufferSize=530904
   [junit4]   2> 1535943 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=53877,localport=42541], receiveBufferSize=530904
   [junit4]   2> 1535943 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40078,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1535944 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53196], receiveBufferSize=530904
   [junit4]   2> 1535948 INFO  (qtp861531069-17929) [n:127.0.0.1:49782_ 
c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 
end_commit_flush
   [junit4]   2> 1535948 INFO  (qtp861531069-17929) [n:127.0.0.1:49782_ 
c:collection1 s:shard1 r:core_node2 x:collection1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:37680/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 8
   [junit4]   2> 1535950 INFO  (qtp448964784-17958) [n:127.0.0.1:52907_ 
c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 
start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1535950 INFO  (qtp950265001-17901) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 
start 
commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1535950 INFO  (qtp448964784-17958) [n:127.0.0.1:52907_ 
c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 
No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1535950 INFO  (qtp950265001-17901) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 
No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1535950 INFO  (qtp448964784-17958) [n:127.0.0.1:52907_ 
c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 
end_commit_flush
   [junit4]   2> 1535951 INFO  (qtp448964784-17958) [n:127.0.0.1:52907_ 
c:collection1 s:shard2 r:core_node3 x:collection1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:37680/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 1
   [junit4]   2> 1535951 INFO  (qtp950265001-17901) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 
end_commit_flush
   [junit4]   2> 1535952 INFO  (qtp950265001-17901) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update 
params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:37680/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=}
 0 1
   [junit4]   2> 1535952 INFO  (qtp950265001-17899) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] 
o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update 
params={_stateVer_=collection1:10&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=}
 0 20
   [junit4]   2> 1535954 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40080,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1535955 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53198], receiveBufferSize=530904
   [junit4]   2> 1535955 INFO  (qtp950265001-17900) [n:127.0.0.1:37680_ 
c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request 
[collection1]  webapp= path=/select 
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
 hits=0 status=0 QTime=0
   [junit4]   2> 1535956 INFO  (SocketProxy-Acceptor-52907) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=36971,localport=52907], receiveBufferSize:531000
   [junit4]   2> 1535956 INFO  (SocketProxy-Acceptor-52907) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=49107,localport=51945], receiveBufferSize=530904
   [junit4]   2> 1535957 INFO  (qtp448964784-17956) [n:127.0.0.1:52907_ 
c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.S.Request 
[collection1]  webapp= path=/select 
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
 hits=0 status=0 QTime=0
   [junit4]   2> 1535958 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=35676,localport=49782], receiveBufferSize:531000
   [junit4]   2> 1535963 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=53877,localport=42550], receiveBufferSize=530904
   [junit4]   2> 1535963 INFO  (qtp861531069-17927) [n:127.0.0.1:49782_ 
c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request 
[collection1]  webapp= path=/select 
params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2}
 hits=0 status=0 QTime=0
   [junit4]   2> 1537966 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40095,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1537966 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=33593,localport=53213], receiveBufferSize=530904
   [junit4]   2> 1537971 INFO  (qtp950265001-17897) [n:127.0.0.1:37680_    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=3&maxShardsPerNode=1&name=c8n_1x3_lf&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1537976 INFO  
(OverseerThreadFactory-6481-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_    ] o.a.s.c.CreateCollectionCmd Create collection 
c8n_1x3_lf
   [junit4]   2> 1537977 INFO  
(OverseerThreadFactory-6481-thread-1-processing-n:127.0.0.1:34191_) 
[n:127.0.0.1:34191_    ] o.a.s.c.CreateCollectionCmd Only one config set found 
in zk - using it:conf1
   [junit4]   2> 1538083 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=35689,localport=49782], receiveBufferSize:531000
   [junit4]   2> 1538084 INFO  (SocketProxy-Acceptor-34191) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=38841,localport=34191], receiveBufferSize:531000
   [junit4]   2> 1538084 INFO  (SocketProxy-Acceptor-37680) [    ] 
o.a.s.c.SocketProxy accepted 
Socket[addr=/127.0.0.1,port=40099,localport=37680], receiveBufferSize:531000
   [junit4]   2> 1538087 INFO  (SocketProxy-Acceptor-49782) [    ] 
o.a.s.c.SocketProxy proxy connection 
Socket[addr=/127.0.0.1,port=53877,localport=42563], receiveBufferSize=530904
 

[...truncated too long message...]

tch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/aliases.json
   [junit4]   2>        5       /solr/clusterprops.json
   [junit4]   2>        4       /solr/security.json
   [junit4]   2>        4       /solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/clusterstate.json
   [junit4]   2>        3       /solr/collections/c8n_1x3_lf/state.json
   [junit4]   2>        3       /solr/collections/collection1/state.json
   [junit4]   2>        2       
/solr/overseer_elect/election/97487507317063684-127.0.0.1:34191_-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        141     /solr/overseer/collection-queue-work
   [junit4]   2>        56      /solr/overseer/queue
   [junit4]   2>        23      /solr/overseer/queue-work
   [junit4]   2>        5       /solr/live_nodes
   [junit4]   2>        5       /solr/collections
   [junit4]   2> 
   [junit4]   2> 1665162 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SocketProxy Closing 16 connections to: http://127.0.0.1:49782/, target: 
http://127.0.0.1:53877/
   [junit4]   2> 1665162 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SocketProxy Closing 2 connections to: http://127.0.0.1:52907/, target: 
http://127.0.0.1:49107/
   [junit4]   2> 1665162 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:37680/, target: 
http://127.0.0.1:33593/
   [junit4]   2> 1665162 WARN  
(TEST-LeaderFailoverAfterPartitionTest.test-seed#[3481232CACD4C33C]) [    ] 
o.a.s.c.SocketProxy Closing 1 connections to: http://127.0.0.1:34191/, target: 
http://127.0.0.1:46890/
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test 
-Dtests.seed=3481232CACD4C33C -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=vi -Dtests.timezone=Etc/GMT0 -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE  145s J2 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Expected 2 of 3 
replicas to be active but only found 1; 
[core_node3:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:49782","node_name":"127.0.0.1:49782_","state":"active","leader":"true"}];
 clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
   [junit4]    >   "replicationFactor":"3",
   [junit4]    >   "shards":{"shard1":{
   [junit4]    >       "range":"80000000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node1":{
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:37680";,
   [junit4]    >           "node_name":"127.0.0.1:37680_",
   [junit4]    >           "state":"down"},
   [junit4]    >         "core_node2":{
   [junit4]    >           "state":"down",
   [junit4]    >           "base_url":"http://127.0.0.1:34191";,
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica1",
   [junit4]    >           "node_name":"127.0.0.1:34191_"},
   [junit4]    >         "core_node3":{
   [junit4]    >           "core":"c8n_1x3_lf_shard1_replica3",
   [junit4]    >           "base_url":"http://127.0.0.1:49782";,
   [junit4]    >           "node_name":"127.0.0.1:49782_",
   [junit4]    >           "state":"active",
   [junit4]    >           "leader":"true"}}}},
   [junit4]    >   "router":{"name":"compositeId"},
   [junit4]    >   "maxShardsPerNode":"1",
   [junit4]    >   "autoAddReplicas":"false"}
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([3481232CACD4C33C:BCD51CF60228AEC4]:0)
   [junit4]    >        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:168)
   [junit4]    >        at 
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:55)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    >        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1665165 INFO  
(SUITE-LeaderFailoverAfterPartitionTest-seed#[3481232CACD4C33C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest_3481232CACD4C33C-001
   [junit4]   2> Feb 19, 2017 9:57:34 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): 
{multiDefault=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), 
id=PostingsFormat(name=LuceneVarGapFixedInterval), text=FSTOrd50, 
timestamp=Lucene50(blocksize=128)}, 
docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), 
_version_=DocValuesFormat(name=Asserting), 
intDefault=DocValuesFormat(name=Asserting), 
range_facet_i_dv=DocValuesFormat(name=Lucene54), 
intDvoDefault=DocValuesFormat(name=Lucene54), 
range_facet_l=DocValuesFormat(name=Lucene54), 
timestamp=DocValuesFormat(name=Lucene54)}, maxPointsInLeafNode=969, 
maxMBSortInHeap=5.174694314187011, 
sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=vi, 
timezone=Etc/GMT0
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_121 (64-bit)/cpus=4,threads=1,free=299267872,total=527958016
   [junit4]   2> NOTE: All tests run in this JVM: 
[OverriddenZkACLAndCredentialsProvidersTest, CustomCollectionTest, 
TestStressLiveNodes, RecoveryZkTest, SimpleCollectionCreateDeleteTest, 
SuggestComponentTest, ManagedSchemaRoundRobinCloudTest, 
WrapperMergePolicyFactoryTest, GraphQueryTest, TestDocTermOrds, 
TestManagedSchemaAPI, FacetPivotSmallTest, TestFieldCacheSanityChecker, 
TestIndexSearcher, UpdateParamsTest, TestDynamicFieldResource, 
DistributedQueryComponentCustomSortTest, SolrGraphiteReporterTest, 
UnloadDistributedZkTest, TestFieldTypeCollectionResource, 
PKIAuthenticationIntegrationTest, OverseerTest, DeleteStatusTest, 
LeaderElectionContextKeyTest, ClassificationUpdateProcessorFactoryTest, 
TestConfigSets, TestCloudSchemaless, TestSmileRequest, 
TestPerFieldSimilarityClassic, DOMUtilTest, CursorMarkTest, 
TestSubQueryTransformerDistrib, TestMaxScoreQueryParser, BadIndexSchemaTest, 
HighlighterMaxOffsetTest, CdcrReplicationHandlerTest, DocValuesMultiTest, 
BlobRepositoryCloudTest, TestCryptoKeys, RollingRestartTest, 
TestClusterStateMutator, TestSolrCloudWithSecureImpersonation, 
SolrMetricReporterTest, HdfsSyncSliceTest, TestGraphMLResponseWriter, 
TestRestoreCore, JvmMetricsTest, TestOrdValues, 
TestOverriddenPrefixQueryForCustomFieldType, TestRecovery, 
CollectionReloadTest, TestGroupingSearch, PingRequestHandlerTest, 
VersionInfoTest, DateRangeFieldTest, TestAnalyzedSuggestions, TestSort, 
ShardSplitTest, QueryParsingTest, TestPivotHelperCode, TestJettySolrRunner, 
ReplaceNodeTest, TestRuleBasedAuthorizationPlugin, TestCollationField, 
TestChildDocTransformer, RequestHandlersTest, StatsComponentTest, 
ParsingFieldUpdateProcessorsTest, CoreAdminHandlerTest, TestCloudRecovery, 
TestMergePolicyConfig, TestCoreDiscovery, TestReplicaProperties, 
SimpleFacetsTest, SuggesterTSTTest, TimeZoneUtilsTest, 
ResponseLogComponentTest, TestReload, SolrPluginUtilsTest, 
TestImpersonationWithHadoopAuth, TestSchemaManager, 
TestPostingsSolrHighlighter, NoCacheHeaderTest, BadCopyFieldTest, 
BasicDistributedZk2Test, MissingSegmentRecoveryTest, 
TestHighFrequencyDictionaryFactory, TestDynamicLoading, TestStressReorder, 
HdfsUnloadDistributedZkTest, JSONWriterTest, DirectUpdateHandlerTest, 
CircularListTest, TestObjectReleaseTracker, CurrencyFieldXmlFileTest, 
BlockJoinFacetDistribTest, TestExtendedDismaxParser, PreAnalyzedFieldTest, 
TestIntervalFaceting, TestSolrJ, TestDownShardTolerantSearch, 
CoreMergeIndexesAdminHandlerTest, FieldAnalysisRequestHandlerTest, 
TestReloadDeadlock, TestSSLRandomization, TemplateUpdateProcessorTest, 
SOLR749Test, TlogReplayBufferedWhileIndexingTest, 
SolrCoreCheckLockOnStartupTest, SparseHLLTest, 
DistributedFacetPivotSmallAdvancedTest, RequestLoggingTest, BasicZkTest, 
FullSolrCloudDistribCmdsTest, TestZkChroot, TestDistributedSearch, 
SchemaVersionSpecificBehaviorTest, FieldMutatingUpdateProcessorTest, 
QueryEqualityTest, DefaultValueUpdateProcessorTest, DistanceFunctionTest, 
SolrInfoMBeanTest, DisMaxRequestHandlerTest, PrimitiveFieldTypeTest, 
FileBasedSpellCheckerTest, MoreLikeThisHandlerTest, LoggingHandlerTest, 
TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, CSVRequestHandlerTest, 
TestBinaryResponseWriter, HighlighterConfigTest, TestQuerySenderListener, 
AlternateDirectoryTest, TestQuerySenderNoQuery, BadComponentTest, 
MultiTermTest, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestConfig, 
DirectSolrConnectionTest, NotRequiredUniqueKeyTest, 
SpellPossibilityIteratorTest, TestCharFilters, TestDFRSimilarityFactory, 
TestJmxMonitoredMap, OpenExchangeRatesOrgProviderTest, TestFastLRUCache, 
PrimUtilsTest, ClusterStateTest, TestUtils, SliceStateTest, FileUtilsTest, 
TestRTGBase, SolrTestCaseJ4Test, TestCrossCoreJoin, 
TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, 
AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, 
CdcrRequestHandlerTest, CleanupOldIndexTest, 
ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, 
ConnectionManagerTest, DeleteNodeTest, DistribJoinFromCollectionTest, 
LeaderFailoverAfterPartitionTest]
   [junit4] Completed [497/696 (1!)] on J2 in 144.88s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 63306 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to