Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/15/

3 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestLargeCluster

Error Message:
1 thread leaked from SUITE scope at 
org.apache.solr.cloud.autoscaling.sim.TestLargeCluster:     1) Thread[id=6290, 
name=AutoscalingActionExecutor-1526-thread-1, state=RUNNABLE, 
group=TGRP-TestLargeCluster]         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)         at 
org.apache.solr.common.util.Utils.makeDeepCopy(Utils.java:108)         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:92)         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)         at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)       
  at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)  
       at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) 
        at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)    
     at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)      
   at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)     
    at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)   
      at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)     
    at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
         at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)         at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         
at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster: 
   1) Thread[id=6290, name=AutoscalingActionExecutor-1526-thread-1, 
state=RUNNABLE, group=TGRP-TestLargeCluster]
        at org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)
        at org.apache.solr.common.util.Utils.makeDeepCopy(Utils.java:108)
        at org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:92)
        at org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)
        at org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)
        at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
        at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)
        at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
        at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
        at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
        at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at 
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
        at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([5858DEE56A6058B5]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestLargeCluster

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=6290, name=AutoscalingActionExecutor-1526-thread-1, state=RUNNABLE, 
group=TGRP-TestLargeCluster]         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)         at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)       
  at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)  
       at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) 
        at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)    
     at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)      
   at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)     
    at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)   
      at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)     
    at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
         at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
         at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)         at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         
at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
        at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=6290, name=AutoscalingActionExecutor-1526-thread-1, 
state=RUNNABLE, group=TGRP-TestLargeCluster]
        at org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)
        at org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)
        at org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)
        at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
        at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)
        at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
        at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
        at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
        at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at 
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
        at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
        at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
        at __randomizedtesting.SeedInfo.seed([5858DEE56A6058B5]:0)


FAILED:  org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN

Error Message:
expected:<1.0> but was:<0.0>

Stack Trace:
java.lang.AssertionError: expected:<1.0> but was:<0.0>
        at 
__randomizedtesting.SeedInfo.seed([1045053CB9E07063:E1E4776C8C5BBAF1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:443)
        at org.junit.Assert.assertEquals(Assert.java:512)
        at 
org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 1772 lines...]
   [junit4] JVM J2: stdout was not empty, see: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/lucene/build/core/test/temp/junit4-J2-20180316_150907_0554641604940642427908.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] codec: SimpleText, pf: LuceneFixedGap, dvf: Asserting
   [junit4] <<< JVM J2: EOF ----

[...truncated 10816 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
   [junit4]   2> 643618 INFO  
(SUITE-TestLargeCluster-seed#[5858DEE56A6058B5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestLargeCluster_5858DEE56A6058B5-001/init-core-data-001
   [junit4]   2> 643620 WARN  
(SUITE-TestLargeCluster-seed#[5858DEE56A6058B5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 643620 INFO  
(SUITE-TestLargeCluster-seed#[5858DEE56A6058B5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 643622 INFO  
(SUITE-TestLargeCluster-seed#[5858DEE56A6058B5]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 643637 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and 
.scheduled_maintenance triggers
   [junit4]   2> 643638 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 643638 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
   [junit4]   2> 643638 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 643638 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 643645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 643645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 643645 DEBUG (ScheduledTrigger-1513-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 14
   [junit4]   2> 643658 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 643658 DEBUG 
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 643659 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 643667 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 643668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 643668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 643669 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 643669 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 643671 DEBUG 
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 643672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and 
.scheduled_maintenance triggers
   [junit4]   2> 643672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 643673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 643673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 643673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 643674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 643674 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 643674 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643695 DEBUG (ScheduledTrigger-1516-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643716 DEBUG (ScheduledTrigger-1516-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643736 DEBUG (ScheduledTrigger-1516-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643756 DEBUG (ScheduledTrigger-1516-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643777 DEBUG (ScheduledTrigger-1516-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643796 DEBUG (simCloudManagerPool-1515-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 643797 DEBUG (simCloudManagerPool-1515-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 643797 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643798 DEBUG (simCloudManagerPool-1515-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 643800 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 643801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 643801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 643817 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643838 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643858 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643878 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643899 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643919 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643940 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643960 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 643982 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644002 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644022 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644031 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.TestLargeCluster Ready after 10244 ms
   [junit4]   2> 644043 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644049 DEBUG (simCloudManagerPool-1515-thread-23) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 644050 DEBUG (simCloudManagerPool-1515-thread-23) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 644055 DEBUG (simCloudManagerPool-1515-thread-23) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644055 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 644056 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644061 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 644063 DEBUG (ScheduledTrigger-1516-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644079 DEBUG (simCloudManagerPool-1515-thread-24) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 644079 DEBUG (simCloudManagerPool-1515-thread-24) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
5
   [junit4]   2> 644080 DEBUG (simCloudManagerPool-1515-thread-24) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644080 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 644080 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644080 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 644083 DEBUG (ScheduledTrigger-1516-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644104 DEBUG (ScheduledTrigger-1516-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644124 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644144 DEBUG (ScheduledTrigger-1516-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644165 DEBUG (ScheduledTrigger-1516-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644185 DEBUG (ScheduledTrigger-1516-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644196 DEBUG (ScheduledTrigger-1516-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"4c446ff1db3b15T76gadqx392v44672srh51ek5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":21467345819745045,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10082_solr":4.0,
   [junit4]   2>       "127.0.0.1:10075_solr":4.0,
   [junit4]   2>       "127.0.0.1:10079_solr":4.0,
   [junit4]   2>       "127.0.0.1:10017_solr":4.0,
   [junit4]   2>       "127.0.0.1:10083_solr":4.0,
   [junit4]   2>       "127.0.0.1:10071_solr":4.0,
   [junit4]   2>       "127.0.0.1:10003_solr":4.0,
   [junit4]   2>       "127.0.0.1:10092_solr":4.0,
   [junit4]   2>       "127.0.0.1:10076_solr":4.0,
   [junit4]   2>       "127.0.0.1:10027_solr":4.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node9\":{\n    
\"core\":\"testSearchRate_shard1_replica_n9\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10082_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"testSearchRate_shard1_replica_n2\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10017_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node6\":{\n    
\"core\":\"testSearchRate_shard1_replica_n6\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10079_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node7\":{\n    
\"core\":\"testSearchRate_shard1_replica_n7\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10003_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node5\":{\n    
\"core\":\"testSearchRate_shard1_replica_n5\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10071_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node4\":{\n    
\"core\":\"testSearchRate_shard1_replica_n4\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10076_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"testSearchRate_shard1_replica_n1\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10083_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node3\":{\n    
\"core\":\"testSearchRate_shard1_replica_n3\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10027_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node10\":{\n    
\"core\":\"testSearchRate_shard1_replica_n10\",\n    \"INDEX.sizeInBytes\":1,\n 
   \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10092_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node8\":{\n    
\"core\":\"testSearchRate_shard1_replica_n8\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10075_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}"],
   [junit4]   2>     "collection":{"testSearchRate":40.0},
   [junit4]   2>     "shard":{"testSearchRate":{"shard1":40.0}},
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}"]}}
   [junit4]   2> 644197 DEBUG (ScheduledTrigger-1516-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, 
search_rate_trigger]
   [junit4]   2> 644208 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"4c446ff1db3b15T76gadqx392v44672srh51ek5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":21467345819745045,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10082_solr":4.0,
   [junit4]   2>       "127.0.0.1:10075_solr":4.0,
   [junit4]   2>       "127.0.0.1:10079_solr":4.0,
   [junit4]   2>       "127.0.0.1:10017_solr":4.0,
   [junit4]   2>       "127.0.0.1:10083_solr":4.0,
   [junit4]   2>       "127.0.0.1:10071_solr":4.0,
   [junit4]   2>       "127.0.0.1:10003_solr":4.0,
   [junit4]   2>       "127.0.0.1:10092_solr":4.0,
   [junit4]   2>       "127.0.0.1:10076_solr":4.0,
   [junit4]   2>       "127.0.0.1:10027_solr":4.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node9\":{\n    
\"core\":\"testSearchRate_shard1_replica_n9\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10082_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"testSearchRate_shard1_replica_n2\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10017_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node6\":{\n    
\"core\":\"testSearchRate_shard1_replica_n6\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10079_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node7\":{\n    
\"core\":\"testSearchRate_shard1_replica_n7\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10003_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node5\":{\n    
\"core\":\"testSearchRate_shard1_replica_n5\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10071_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node4\":{\n    
\"core\":\"testSearchRate_shard1_replica_n4\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10076_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"testSearchRate_shard1_replica_n1\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10083_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node3\":{\n    
\"core\":\"testSearchRate_shard1_replica_n3\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10027_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node10\":{\n    
\"core\":\"testSearchRate_shard1_replica_n10\",\n    \"INDEX.sizeInBytes\":1,\n 
   \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10092_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node8\":{\n    
\"core\":\"testSearchRate_shard1_replica_n8\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10075_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}"],
   [junit4]   2>     "collection":{"testSearchRate":40.0},
   [junit4]   2>     "shard":{"testSearchRate":{"shard1":40.0}},
   [junit4]   2>     "_enqueue_time_":21467352495869545,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}"]}}
   [junit4]   2> 644214 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"4c446ff1db3b15T76gadqx392v44672srh51ek5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":21467345819745045,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10082_solr":4.0,
   [junit4]   2>       "127.0.0.1:10075_solr":4.0,
   [junit4]   2>       "127.0.0.1:10079_solr":4.0,
   [junit4]   2>       "127.0.0.1:10017_solr":4.0,
   [junit4]   2>       "127.0.0.1:10083_solr":4.0,
   [junit4]   2>       "127.0.0.1:10071_solr":4.0,
   [junit4]   2>       "127.0.0.1:10003_solr":4.0,
   [junit4]   2>       "127.0.0.1:10092_solr":4.0,
   [junit4]   2>       "127.0.0.1:10076_solr":4.0,
   [junit4]   2>       "127.0.0.1:10027_solr":4.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node9\":{\n    
\"core\":\"testSearchRate_shard1_replica_n9\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10082_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"testSearchRate_shard1_replica_n2\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10017_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node6\":{\n    
\"core\":\"testSearchRate_shard1_replica_n6\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10079_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node7\":{\n    
\"core\":\"testSearchRate_shard1_replica_n7\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10003_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node5\":{\n    
\"core\":\"testSearchRate_shard1_replica_n5\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10071_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node4\":{\n    
\"core\":\"testSearchRate_shard1_replica_n4\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10076_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"testSearchRate_shard1_replica_n1\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10083_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node3\":{\n    
\"core\":\"testSearchRate_shard1_replica_n3\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10027_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node10\":{\n    
\"core\":\"testSearchRate_shard1_replica_n10\",\n    \"INDEX.sizeInBytes\":1,\n 
   \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10092_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node8\":{\n    
\"core\":\"testSearchRate_shard1_replica_n8\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10075_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}"],
   [junit4]   2>     "collection":{"testSearchRate":40.0},
   [junit4]   2>     "shard":{"testSearchRate":{"shard1":40.0}},
   [junit4]   2>     "_enqueue_time_":21467352495869545,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}"]}} with context 
properties: {BEFORE_ACTION=[compute]}
   [junit4]   2> 644280 INFO  (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction Computed Plan: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10072_solr&type=NRT
   [junit4]   2> 644312 INFO  (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction Computed Plan: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10059_solr&type=NRT
   [junit4]   2> 644375 INFO  (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ComputePlanAction Computed Plan: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10053_solr&type=NRT
   [junit4]   2> 644375 INFO  (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 21467361337 
sessionWrapper.createTime 21467353804810345, this.sessionWrapper.createTime 
21467353804810345 
   [junit4]   2> 644376 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"4c446ff1db3b15T76gadqx392v44672srh51ek5l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":21467345819745045,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10082_solr":4.0,
   [junit4]   2>       "127.0.0.1:10075_solr":4.0,
   [junit4]   2>       "127.0.0.1:10079_solr":4.0,
   [junit4]   2>       "127.0.0.1:10017_solr":4.0,
   [junit4]   2>       "127.0.0.1:10083_solr":4.0,
   [junit4]   2>       "127.0.0.1:10071_solr":4.0,
   [junit4]   2>       "127.0.0.1:10003_solr":4.0,
   [junit4]   2>       "127.0.0.1:10092_solr":4.0,
   [junit4]   2>       "127.0.0.1:10076_solr":4.0,
   [junit4]   2>       "127.0.0.1:10027_solr":4.0},
   [junit4]   2>     "__start__":1,
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node9\":{\n    
\"core\":\"testSearchRate_shard1_replica_n9\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10082_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"testSearchRate_shard1_replica_n2\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10017_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node6\":{\n    
\"core\":\"testSearchRate_shard1_replica_n6\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10079_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node7\":{\n    
\"core\":\"testSearchRate_shard1_replica_n7\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10003_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node5\":{\n    
\"core\":\"testSearchRate_shard1_replica_n5\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10071_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node4\":{\n    
\"core\":\"testSearchRate_shard1_replica_n4\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10076_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"testSearchRate_shard1_replica_n1\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10083_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node3\":{\n    
\"core\":\"testSearchRate_shard1_replica_n3\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10027_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node10\":{\n    
\"core\":\"testSearchRate_shard1_replica_n10\",\n    \"INDEX.sizeInBytes\":1,\n 
   \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10092_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}",
   [junit4]   2>       "{\"core_node8\":{\n    
\"core\":\"testSearchRate_shard1_replica_n8\",\n    \"INDEX.sizeInBytes\":1,\n  
  \"rate\":4.0,\n    \"node_name\":\"127.0.0.1:10075_solr\",\n    
\"state\":\"active\",\n    \"type\":\"NRT\",\n    
\"QUERY./select.requestTimes:1minRate\":4.0,\n    \"shard\":\"shard1\",\n    
\"collection\":\"testSearchRate\"}}"],
   [junit4]   2>     "collection":{"testSearchRate":40.0},
   [junit4]   2>     "shard":{"testSearchRate":{"shard1":40.0}},
   [junit4]   2>     "_enqueue_time_":21467352495869545,
   [junit4]   2>     "requestedOps":[
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}",
   [junit4]   2>       "Op{action=ADDREPLICA, hints={COLL_SHARD={\n  
\"first\":\"testSearchRate\",\n  \"second\":\"shard1\"}}}"]}} with context 
properties: 
{operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@463e96ea,
 
org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@5e9c653, 
org.apache.solr.client.solrj.request.CollectionAdminRequest$AddReplica@1e55b45f],
 AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
   [junit4]   2> 644377 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10072_solr&type=NRT
   [junit4]   2> 644389 INFO  (simCloudManagerPool-1515-thread-26) [    ] 
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10072_solr for creating new 
replica
   [junit4]   2> 644409 DEBUG (simCloudManagerPool-1515-thread-27) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / 
shard2: {"core_node18":{
   [junit4]   2>     "node_name":"127.0.0.1:10029_solr",
   [junit4]   2>     "core":"testSearchRate_shard2_replica_n18",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "shard":"shard2",
   [junit4]   2>     "collection":"testSearchRate"}}
   [junit4]   2> 644412 DEBUG (simCloudManagerPool-1515-thread-28) [    ] 
o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testSearchRate / 
shard1: {"core_node1":{
   [junit4]   2>     "core":"testSearchRate_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "INDEX.sizeInBytes":1,
   [junit4]   2>     "rate":4.0,
   [junit4]   2>     "node_name":"127.0.0.1:10083_solr",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "QUERY./select.requestTimes:1minRate":4.0,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testSearchRate"}}
   [junit4]   2> 644441 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10059_solr&type=NRT
   [junit4]   2> 644452 INFO  (simCloudManagerPool-1515-thread-31) [    ] 
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10059_solr for creating new 
replica
   [junit4]   2> 644464 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ExecutePlanAction Executing operation: 
action=ADDREPLICA&collection=testSearchRate&shard=shard1&node=127.0.0.1:10053_solr&type=NRT
   [junit4]   2> 644468 INFO  (simCloudManagerPool-1515-thread-36) [    ] 
o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:10053_solr for creating new 
replica
   [junit4]   2> 644533 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
   [junit4]   2> 644533 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Resuming trigger: search_rate_trigger after 100ms
   [junit4]   2> 644533 DEBUG (AutoscalingActionExecutor-1517-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 16326 ms for event 
id=4c446ff1db3b15T76gadqx392v44672srh51ek5l
   [junit4]   2> 644566 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
   [junit4]   2> 644566 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 644566 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 644566 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 644566 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 644567 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:             100
   [junit4]   2> 644568 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:    77
   [junit4]   2> 644568 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:             0
   [junit4]   2> 644569 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:    [testSearchRate]
   [junit4]   2> 644569 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:  1
   [junit4]   2> 644569 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:  0
   [junit4]   2> 644569 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:         23
   [junit4]   2> 644570 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * testSearchRate                23
   [junit4]   2> 644571 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - active          23
   [junit4]   2> 644571 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 644571 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - ADDREPLICA         3
   [junit4]   2> 644571 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - CREATE             1
   [junit4]   2> 644575 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - DELETESTATUS       3
   [junit4]   2> 644575 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - REQUESTSTATUS      3
   [junit4]   2> 644575 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - autoscaling        3
   [junit4]   2> 644575 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - update             2
   [junit4]   2> 644575 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 644576 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: search_rate_trigger
   [junit4]   2> 644577 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - STARTED         1
   [junit4]   2> 644577 INFO  
(TEST-TestLargeCluster.testSearchRate-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##         - SUCCEEDED       1
   [junit4]   2> 644604 INFO  
(TEST-TestLargeCluster.testNodeLost-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
   [junit4]   2> 644605 DEBUG 
(TEST-TestLargeCluster.testNodeLost-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
6
   [junit4]   2> 644605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 644605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 644605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 644605 INFO  
(TEST-TestLargeCluster.testNodeLost-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 644606 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 644606 DEBUG 
(TEST-TestLargeCluster.testNodeLost-seed#[5858DEE56A6058B5]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 644613 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and 
.scheduled_maintenance triggers
   [junit4]   2> 644615 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 7
   [junit4]   2> 644616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
   [junit4]   2> 644616 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 644619 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644619 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 644619 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644646 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644667 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644687 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644707 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644711 DEBUG (simCloudManagerPool-1518-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 644711 DEBUG (simCloudManagerPool-1518-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 644715 DEBUG (simCloudManagerPool-1518-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644715 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 644716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 644719 DEBUG (simCloudManagerPool-1518-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 644719 DEBUG (simCloudManagerPool-1518-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
9
   [junit4]   2> 644719 DEBUG (simCloudManagerPool-1518-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644719 DEBUG (simCloudManagerPool-1518-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 
127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 
127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 
127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 
127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 
127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 
127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 
127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 
127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 
127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 
127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 
127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 
127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 
127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 
127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 
127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 
127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 
127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 
127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 
127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 
127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 
127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 
127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 
127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 
127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 
127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 
127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 
127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 
127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 
127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 
127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 
127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 
127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 
127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
   [junit4]   2> 644720 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 644720 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 644720 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 644720 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644728 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644741 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644748 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644761 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644769 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644782 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644789 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644802 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644809 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644822 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644830 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644843 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644851 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644863 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644871 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644883 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644891 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644904 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644912 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644924 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644932 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644944 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644952 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644965 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644973 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 644985 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 644993 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645005 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645013 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645026 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645034 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645046 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645054 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645067 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645074 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645087 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645095 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645107 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645115 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645128 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645135 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645148 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645156 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645168 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645176 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645189 DEBUG (ScheduledTrigger-1519-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645196 DEBUG (ScheduledTrigger-1519-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645209 DEBUG (ScheduledTrigger-1519-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 100
   [junit4]   2> 645217 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 100
   [junit4]   2> 645230 DEBUG (ScheduledTrigger-1519-thread-2) [    ] 
o.a.s.c.a.Node

[...truncated too long message...]

 TimeZoneUtilsTest, TestUtils, DocExpirationUpdateProcessorFactoryTest, 
DistributedFacetPivotLongTailTest, ZkFailoverTest, BooleanFieldTest, 
TestDistributedMissingSort, TestUnifiedSolrHighlighter, 
RecoveryAfterSoftCommitTest, TestJettySolrRunner, DistanceFunctionTest, 
TestLargeCluster]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLargeCluster 
-Dtests.seed=5858DEE56A6058B5 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.badapples=true -Dtests.locale=hr-HR -Dtests.timezone=Turkey 
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | TestLargeCluster (suite) <<<
   [junit4]    > Throwable #1: 
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE 
scope at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster: 
   [junit4]    >    1) Thread[id=6290, 
name=AutoscalingActionExecutor-1526-thread-1, state=RUNNABLE, 
group=TGRP-TestLargeCluster]
   [junit4]    >         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)
   [junit4]    >         at 
org.apache.solr.common.util.Utils.makeDeepCopy(Utils.java:108)
   [junit4]    >         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:92)
   [junit4]    >         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)
   [junit4]    >         at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
   [junit4]    >         at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
   [junit4]    >         at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
   [junit4]    >         at 
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)
   [junit4]    >         at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]    >         at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]    >         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([5858DEE56A6058B5]:0)Throwable #2: 
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=6290, 
name=AutoscalingActionExecutor-1526-thread-1, state=RUNNABLE, 
group=TGRP-TestLargeCluster]
   [junit4]    >         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:90)
   [junit4]    >         at 
org.apache.solr.common.util.Utils.getDeepCopy(Utils.java:74)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.copy(Row.java:91)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.lambda$getMatrixCopy$1(Policy.java:299)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session$$Lambda$666/1710382526.apply(Unknown
 Source)
   [junit4]    >         at 
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
   [junit4]    >         at 
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1374)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
   [junit4]    >         at 
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
   [junit4]    >         at 
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
   [junit4]    >         at 
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.getMatrixCopy(Policy.java:300)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Policy$Session.copy(Policy.java:289)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Row.addReplica(Row.java:122)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.tryEachNode(MoveReplicaSuggester.java:59)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.init(MoveReplicaSuggester.java:34)
   [junit4]    >         at 
org.apache.solr.client.solrj.cloud.autoscaling.Suggester.getSuggestion(Suggester.java:129)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ComputePlanAction.process(ComputePlanAction.java:98)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:308)
   [junit4]    >         at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$658/1726392129.run(Unknown
 Source)
   [junit4]    >         at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]    >         at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]    >         at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$125/1908389490.run(Unknown
 Source)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    >         at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([5858DEE56A6058B5]:0)
   [junit4] Completed [163/784 (1!)] on J2 in 63.93s, 4 tests, 2 errors <<< 
FAILURES!

[...truncated 9467 lines...]
   [junit4] Suite: org.apache.solr.ltr.TestLTRReRankingPipeline
   [junit4]   2> 20620 INFO  
(TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[1045053CB9E07063]) [    
] o.a.s.l.TestLTRReRankingPipeline rerank 1 documents 
   [junit4]   2> 20621 INFO  
(TEST-TestLTRReRankingPipeline.testDifferentTopN-seed#[1045053CB9E07063]) [    
] o.a.s.l.TestLTRReRankingPipeline doc 0 in pos 0
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestLTRReRankingPipeline -Dtests.method=testDifferentTopN 
-Dtests.seed=1045053CB9E07063 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.badapples=true -Dtests.locale=ja-JP -Dtests.timezone=PRT 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.05s J0 | TestLTRReRankingPipeline.testDifferentTopN <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<1.0> but 
was:<0.0>
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([1045053CB9E07063:E1E4776C8C5BBAF1]:0)
   [junit4]    >        at 
org.apache.solr.ltr.TestLTRReRankingPipeline.testDifferentTopN(TestLTRReRankingPipeline.java:255)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-Tests-7.x/solr/build/contrib/solr-ltr/test/J0/temp/solr.ltr.TestLTRReRankingPipeline_1045053CB9E07063-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): 
{field=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))),
 id=FST50, final-score=FST50}, 
docValues:{final-score=DocValuesFormat(name=Lucene70)}, 
maxPointsInLeafNode=1287, maxMBSortInHeap=7.9217326155984775, 
sim=RandomSimilarity(queryNorm=true): {field=DFR I(n)LZ(0.3)}, locale=ja-JP, 
timezone=PRT
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 
1.8.0_144 (64-bit)/cpus=4,threads=1,free=243612928,total=326631424
   [junit4]   2> NOTE: All tests run in this JVM: [TestSelectiveWeightCreation, 
TestUserTermScorereQDF, TestLTRWithSort, TestDefaultWrapperModel, 
TestFieldLengthFeature, TestNeuralNetworkModel, TestLTRReRankingPipeline]
   [junit4] Completed [18/37 (1!)] on J0 in 0.15s, 3 tests, 1 failure <<< 
FAILURES!

[...truncated 42062 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to