[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 165 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/165/

3 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode

Error Message:
no MOVEREPLICA ops?

Stack Trace:
java.lang.AssertionError: no MOVEREPLICA ops?
        at 
__randomizedtesting.SeedInfo.seed([EC5794D1006C94B6:4BB88972CF211BAE]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:262)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
        at 
__randomizedtesting.SeedInfo.seed([EC5794D1006C94B6:53425A2F8386F130]:0)
        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.waitForState(SimSolrCloudTestCase.java:261)
        at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.doTestNodeLost(TestLargeCluster.java:456)
        at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost(TestLargeCluster.java:298)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic

Error Message:
there should be new MOVERPLICA ops

Stack Trace:
java.lang.AssertionError: there should be new MOVERPLICA ops
        at 
__randomizedtesting.SeedInfo.seed([EC5794D1006C94B6:47AD89C4DFB01298]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testBasic(TestLargeCluster.java:206)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 14127 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
   [junit4]   2> 4576536 INFO  
(SUITE-TestLargeCluster-seed#[EC5794D1006C94B6]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_EC5794D1006C94B6-001/init-core-data-001
   [junit4]   2> 4576601 WARN  
(SUITE-TestLargeCluster-seed#[EC5794D1006C94B6]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=165 numCloses=165
   [junit4]   2> 4576602 INFO  
(SUITE-TestLargeCluster-seed#[EC5794D1006C94B6]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4576603 INFO  
(SUITE-TestLargeCluster-seed#[EC5794D1006C94B6]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4] IGNOR/A 0.00s J1 | TestLargeCluster.testSearchRate
   [junit4]    > Assumption #1: 'badapple' test group is disabled 
(@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-11714))
   [junit4]   2> 4576872 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 4576872 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 4576872 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 4576873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 4576873 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 4577060 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4577060 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 4577100 INFO  
(TEST-TestLargeCluster.testAddNode-seed#[EC5794D1006C94B6]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testAddNode
   [junit4]   2> 4577113 DEBUG (ScheduledTrigger-5474-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577114 DEBUG 
(TEST-TestLargeCluster.testAddNode-seed#[EC5794D1006C94B6]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 4577133 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 4577223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 4577224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4577224 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 4577225 INFO  
(TEST-TestLargeCluster.testAddNode-seed#[EC5794D1006C94B6]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 4577225 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 4577246 DEBUG 
(TEST-TestLargeCluster.testAddNode-seed#[EC5794D1006C94B6]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 4577258 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 4577259 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 4577272 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 4577272 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 4577272 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 4577309 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 4577309 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 4577309 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577330 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577423 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577585 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577618 DEBUG (simCloudManagerPool-5476-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 4577618 DEBUG (simCloudManagerPool-5476-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 4577637 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577653 DEBUG (simCloudManagerPool-5476-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 4577656 DEBUG (simCloudManagerPool-5476-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 4577656 DEBUG (simCloudManagerPool-5476-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=5, actions=[{name=compute, 
class=org.apache.solr.cloud.autoscaling.ComputePlanAction}, {name=execute, 
class=org.apache.solr.cloud.autoscaling.ExecutePlanAction}], enabled=true}
   [junit4]   2> 4577656 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 4577657 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577832 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 4577832 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577852 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577672 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4577853 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4577889 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4577942 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578105 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578126 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578128 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578129 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578161 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578161 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578161 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578197 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578197 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578198 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578234 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578234 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578234 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578270 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578271 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578271 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578307 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578307 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578307 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578331 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578331 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578331 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578355 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578355 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578355 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578388 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578388 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578388 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578425 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578425 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578425 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578449 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578449 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578450 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578470 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578470 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578470 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578506 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578506 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578506 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578543 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578543 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578543 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578580 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578580 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578580 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578632 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578632 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578632 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578660 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578661 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578661 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578681 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578681 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578681 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578723 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578723 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578723 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578743 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578743 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578743 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578778 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578778 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578778 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4578811 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4578811 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4578811 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579619 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579619 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579619 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579639 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579639 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579639 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579676 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579676 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579678 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579712 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579712 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579712 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579749 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579750 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579750 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579785 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579785 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579785 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579822 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579822 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579822 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579858 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579859 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579859 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579895 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579895 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579895 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579931 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579932 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579932 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4579968 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4579968 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4579968 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580005 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580005 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580005 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580037 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580037 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580037 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580057 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580058 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580058 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580078 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580078 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580078 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580110 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580110 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580110 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580131 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580131 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580131 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580167 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580167 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580167 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580204 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580204 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580204 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580240 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580240 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580240 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580277 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580277 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580277 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580313 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580313 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580314 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580350 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580350 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580350 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580386 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580387 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580387 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580423 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580423 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580423 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580459 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580460 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580460 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580496 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580496 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580496 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580533 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580533 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580533 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580569 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580569 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580569 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580606 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580606 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580606 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580643 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580643 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580643 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580681 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580681 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580681 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580734 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580734 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580734 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580734 DEBUG (simCloudManagerPool-5476-thread-12) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard3: {"core_node71":{
   [junit4]   2>     "node_name":"127.0.0.1:10095_solr",
   [junit4]   2>     "core":"testNodeAdded_shard3_replica_t71",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"TLOG",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard3",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580773 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580773 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580773 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580789 DEBUG (simCloudManagerPool-5476-thread-11) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard2: {"core_node33":{
   [junit4]   2>     "node_name":"127.0.0.1:10084_solr",
   [junit4]   2>     "core":"testNodeAdded_shard2_replica_n33",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580792 DEBUG (simCloudManagerPool-5476-thread-11) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard7: {"core_node191":{
   [junit4]   2>     "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>     "core":"testNodeAdded_shard7_replica_t191",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"TLOG",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard7",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580792 DEBUG (simCloudManagerPool-5476-thread-12) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard1: {"core_node18":{
   [junit4]   2>     "node_name":"127.0.0.1:10099_solr",
   [junit4]   2>     "core":"testNodeAdded_shard1_replica_t18",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"TLOG",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580792 DEBUG (simCloudManagerPool-5476-thread-8) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard5: {"core_node129":{
   [junit4]   2>     "node_name":"127.0.0.1:10035_solr",
   [junit4]   2>     "core":"testNodeAdded_shard5_replica_n129",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard5",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580809 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580810 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580810 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580809 DEBUG (simCloudManagerPool-5476-thread-13) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / 
shard6: {"core_node168":{
   [junit4]   2>     "node_name":"127.0.0.1:10082_solr",
   [junit4]   2>     "core":"testNodeAdded_shard6_replica_t168",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"TLOG",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard6",
   [junit4]   2>     "collection":"testNodeAdded"}}
   [junit4]   2> 4580830 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580830 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580830 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580862 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580862 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580862 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580882 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580882 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580882 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580902 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580903 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580903 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580927 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580927 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580927 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580947 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580947 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580947 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4580980 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4580980 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4580980 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581016 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581016 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581016 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581052 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581052 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581053 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581077 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581077 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581078 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581098 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581098 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581098 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581118 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581118 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581118 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581139 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581139 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581139 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581159 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581159 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581159 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581179 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581179 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581179 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581201 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581201 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581201 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581222 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581222 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581222 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581242 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581242 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581242 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581262 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581262 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581262 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581282 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581283 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581283 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581316 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581316 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581316 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581353 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581353 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581353 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581389 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581389 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581389 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581426 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581426 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581426 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581462 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581462 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581463 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581499 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581499 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581499 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581532 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581532 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581532 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581552 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581552 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581552 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581588 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581588 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581588 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581625 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581625 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581625 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581656 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581657 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581657 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581677 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581677 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581677 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581719 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581719 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581719 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581739 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581739 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581739 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581776 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581776 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581776 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581815 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581815 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581815 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581836 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581836 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581836 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581864 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581864 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581886 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581886 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581886 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581922 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581922 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581922 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581954 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581954 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581954 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581975 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581975 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581975 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4581995 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4581995 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4581995 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582047 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582048 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582048 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582068 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582068 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582068 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582104 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582104 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582104 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582145 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582145 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582145 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582177 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582177 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582177 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582214 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582214 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582214 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582251 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582251 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582251 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582303 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582303 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582303 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582323 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582324 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582324 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582356 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582356 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582356 DEBUG (ScheduledTrigger-5477-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582376 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582376 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582376 DEBUG (ScheduledTrigger-5477-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582409 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582409 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582409 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582445 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582445 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582445 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582482 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582482 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582482 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582520 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582520 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582520 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582551 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582551 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582551 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582583 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582583 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582583 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582603 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582604 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582604 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582650 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582650 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582650 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582670 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582671 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582671 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582691 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582691 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582691 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582711 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582711 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582711 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582732 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582732 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582732 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582753 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582753 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582753 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582773 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582773 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582773 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582793 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582793 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582793 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582813 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582814 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582814 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582851 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582851 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582851 DEBUG (ScheduledTrigger-5477-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582884 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582884 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582884 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582910 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582910 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582910 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   2> 4582944 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 4582945 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 4582945 DEBUG (ScheduledTrigger-5477-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
   [junit4]   

[...truncated too long message...]

Plan: 
action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10116_solr&inPlaceMove=true&replica=core_node20
   [junit4]   2> 4649539 INFO  (AutoscalingActionExecutor-5484-thread-1) [    ] 
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2519571065 
sessionWrapper.createTime 2519448593323367, this.sessionWrapper.createTime 
2519448593323367 
   [junit4]   2> 4649539 WARN  (AutoscalingActionExecutor-5484-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> java.lang.Exception: Error executing action: execute for 
trigger event: {
   [junit4]   2>   "id":"8f36b9e5c8c95T6gqon3mtg9qdlieas8sjh9wtr",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2519443357600917,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[
   [junit4]   2>       2519443357600917,
   [junit4]   2>       2519443350715017,
   [junit4]   2>       2519443336858817,
   [junit4]   2>       2519443361032017,
   [junit4]   2>       2519443340362817,
   [junit4]   2>       2519443354164867,
   [junit4]   2>       2519443333363367,
   [junit4]   2>       2519443343822117,
   [junit4]   2>       2519443364493717,
   [junit4]   2>       2519443347287417],
   [junit4]   2>     "_enqueue_time_":2519448525680367,
   [junit4]   2>     "nodeNames":[
   [junit4]   2>       "127.0.0.1:10139_solr",
   [junit4]   2>       "127.0.0.1:10016_solr",
   [junit4]   2>       "127.0.0.1:10054_solr",
   [junit4]   2>       "127.0.0.1:10133_solr",
   [junit4]   2>       "127.0.0.1:10123_solr",
   [junit4]   2>       "127.0.0.1:10077_solr",
   [junit4]   2>       "127.0.0.1:10014_solr",
   [junit4]   2>       "127.0.0.1:10071_solr",
   [junit4]   2>       "127.0.0.1:10114_solr",
   [junit4]   2>       "127.0.0.1:10107_solr"]}}
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:307)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected 
exception while processing event: {
   [junit4]   2>   "id":"8f36b9e5c8c95T6gqon3mtg9qdlieas8sjh9wtr",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2519443357600917,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[
   [junit4]   2>       2519443357600917,
   [junit4]   2>       2519443350715017,
   [junit4]   2>       2519443336858817,
   [junit4]   2>       2519443361032017,
   [junit4]   2>       2519443340362817,
   [junit4]   2>       2519443354164867,
   [junit4]   2>       2519443333363367,
   [junit4]   2>       2519443343822117,
   [junit4]   2>       2519443364493717,
   [junit4]   2>       2519443347287417],
   [junit4]   2>     "_enqueue_time_":2519448525680367,
   [junit4]   2>     "nodeNames":[
   [junit4]   2>       "127.0.0.1:10139_solr",
   [junit4]   2>       "127.0.0.1:10016_solr",
   [junit4]   2>       "127.0.0.1:10054_solr",
   [junit4]   2>       "127.0.0.1:10133_solr",
   [junit4]   2>       "127.0.0.1:10123_solr",
   [junit4]   2>       "127.0.0.1:10077_solr",
   [junit4]   2>       "127.0.0.1:10014_solr",
   [junit4]   2>       "127.0.0.1:10071_solr",
   [junit4]   2>       "127.0.0.1:10114_solr",
   [junit4]   2>       "127.0.0.1:10107_solr"]}}
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:304)
   [junit4]   2>        ... 6 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unexpected 
exception executing operation: 
action=MOVEREPLICA&async=node_lost_trigger/8f36b9e5c8c95T6gqon3mtg9qdlieas8sjh9wtr/0&waitForFinalState=true&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node59
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:110)
   [junit4]   2>        ... 7 more
   [junit4]   2> Caused by: java.io.IOException: 
java.util.concurrent.RejectedExecutionException: Task 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$121/696547655@6786b188
 rejected from 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@458454b0[Shutting
 down, pool size = 24, active threads = 0, queued tasks = 0, completed tasks = 
1205]
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:462)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:80)
   [junit4]   2>        ... 7 more
   [junit4]   2> Caused by: java.util.concurrent.RejectedExecutionException: 
Task 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$121/696547655@6786b188
 rejected from 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@458454b0[Shutting
 down, pool size = 24, active threads = 0, queued tasks = 0, completed tasks = 
1205]
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:173)
   [junit4]   2>        at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.sim.SimCloudManager.submit(SimCloudManager.java:382)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:459)
   [junit4]   2>        ... 8 more
   [junit4]   2> Mar 04, 2018 3:49:56 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 25 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=736, maxMBSortInHeap=6.080682270611448, 
sim=RandomSimilarity(queryNorm=true): {}, locale=it-IT, timezone=MET
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=127162016,total=535822336
   [junit4]   2> NOTE: All tests run in this JVM: [TestPostingsSolrHighlighter, 
QueryParsingTest, TestLeaderElectionWithEmptyReplica, 
DistributedExpandComponentTest, HdfsNNFailoverTest, TestBulkSchemaAPI, 
SolrCloudExampleTest, TestNumericTerms32, TestAnalyzeInfixSuggestions, 
TestMacros, TestHdfsCloudBackupRestore, MoveReplicaHDFSFailoverTest, 
DirectoryFactoryTest, TestLegacyFieldReuse, TestSolrCloudWithDelegationTokens, 
TestQueryUtils, SchemaVersionSpecificBehaviorTest, 
TestSha256AuthenticationProvider, TestCharFilters, TestOrdValues, 
TestTrackingShardHandlerFactory, TestBM25SimilarityFactory, ZkStateWriterTest, 
DocValuesNotIndexedTest, TestDocBasedVersionConstraints, 
TestSerializedLuceneMatchVersion, TestDynamicFieldResource, 
DistribJoinFromCollectionTest, CoreAdminCreateDiscoverTest, TestLockTree, 
TestSearchPerf, TestScoreJoinQPNoScore, CollectionsAPIAsyncDistributedZkTest, 
TestTrie, TestCorePropertiesReload, SuggesterTest, SoftAutoCommitTest, 
BlockJoinFacetDistribTest, TestBlobHandler, MultiThreadedOCPTest, BasicZkTest, 
DirectSolrConnectionTest, ResponseHeaderTest, TestZkChroot, 
TestFieldTypeCollectionResource, HdfsBasicDistributedZkTest, 
TestMultiValuedNumericRangeQuery, TestHdfsBackupRestoreCore, TestConfigSetsAPI, 
TestRandomFlRTGCloud, BlockDirectoryTest, HLLUtilTest, 
SuggestComponentContextFilterQueryTest, TestJsonFacetRefinement, 
SolrMetricReporterTest, TestDistributedMap, FileUtilsTest, 
TestUnifiedSolrHighlighter, DistributedFacetPivotWhiteBoxTest, 
CustomCollectionTest, WrapperMergePolicyFactoryTest, CloudMLTQParserTest, 
TestReRankQParserPlugin, TestStressCloudBlindAtomicUpdates, InfoHandlerTest, 
LoggingHandlerTest, TestSolrCloudWithHadoopAuthPlugin, SolrInfoBeanTest, 
ReplaceNodeNoTargetTest, TestSolrJ, LeaderFailoverAfterPartitionTest, 
TestManagedSynonymFilterFactory, TestSolr4Spatial, 
FieldMutatingUpdateProcessorTest, BasicAuthIntegrationTest, 
TestSolrConfigHandlerCloud, TestPerFieldSimilarity, TestJettySolrRunner, 
TestExpandComponent, ReplicaListTransformerTest, TestReplicationHandler, 
XsltUpdateRequestHandlerTest, AlternateDirectoryTest, 
IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestFilteredDocIdSet, 
ZkControllerTest, TestReversedWildcardFilterFactory, TestUtilizeNode, 
TestSimpleTrackingShardHandler, TestCursorMarkWithoutUniqueKey, 
TestStandardQParsers, SampleTest, TestShardHandlerFactory, 
FastVectorHighlighterTest, TestSolr4Spatial2, QueryElevationComponentTest, 
BigEndianAscendingWordDeserializerTest, TimeRoutedAliasUpdateProcessorTest, 
QueryEqualityTest, TestInitParams, SolrGraphiteReporterTest, 
TestEmbeddedSolrServerAdminHandler, TestShortCircuitedRequests, 
AsyncCallRequestStatusResponseTest, AtomicUpdatesTest, 
TestManagedSchemaThreadSafety, TestCloudConsistency, TestRestoreCore, 
DistributedIntervalFacetingTest, TestLRUCache, TestSearcherReuse, 
TestAddFieldRealTimeGet, PeerSyncTest, 
TestOverriddenPrefixQueryForCustomFieldType, TestFieldCacheVsDocValues, 
TestRecovery, TestLargeCluster]
   [junit4] Completed [392/779 (1!)] on J1 in 73.42s, 4 tests, 2 failures, 1 
error, 1 skipped <<< FAILURES!

[...truncated 49251 lines...]

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org