Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1587/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
        at 
__randomizedtesting.SeedInfo.seed([EA8383113195D55C:11A12B34E33F36CE]:0)
        at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
        at java.util.ArrayList$Itr.next(ArrayList.java:859)
        at 
org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
        at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12519 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> 1767518 INFO  
(SUITE-TestTriggerIntegration-seed#[EA8383113195D55C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_EA8383113195D55C-001/init-core-data-001
   [junit4]   2> 1767519 WARN  
(SUITE-TestTriggerIntegration-seed#[EA8383113195D55C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1767519 INFO  
(SUITE-TestTriggerIntegration-seed#[EA8383113195D55C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1767521 INFO  
(SUITE-TestTriggerIntegration-seed#[EA8383113195D55C]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1767522 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1767522 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 1767522 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767522 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1767522 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1767528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767528 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 1767528 DEBUG (ScheduledTrigger-5425-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767537 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 1767537 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 1767540 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 1767537 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1767547 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1767548 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767548 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1767548 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1767549 DEBUG (ScheduledTrigger-5428-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767570 DEBUG (ScheduledTrigger-5428-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767590 DEBUG (ScheduledTrigger-5428-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767615 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767635 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767656 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767676 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767693 DEBUG (simCloudManagerPool-5427-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1767693 DEBUG (simCloudManagerPool-5427-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 1767695 DEBUG (simCloudManagerPool-5427-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767695 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1767696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1767696 DEBUG (ScheduledTrigger-5428-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767698 DEBUG (simCloudManagerPool-5427-thread-5) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1767698 DEBUG (simCloudManagerPool-5427-thread-5) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
4
   [junit4]   2> 1767698 DEBUG (simCloudManagerPool-5427-thread-5) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767699 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1767700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1767717 DEBUG (ScheduledTrigger-5428-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767741 DEBUG (ScheduledTrigger-5428-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767742 DEBUG (ScheduledTrigger-5428-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"781508b1f1f12T5knc7tf4zo2cv61rh75vg392g",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":2112507768413970,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10006_solr":250.0,
   [junit4]   2>       "127.0.0.1:10007_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10006_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10007_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}}}}
   [junit4]   2> 1767746 DEBUG (AutoscalingActionExecutor-5429-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"781508b1f1f12T5knc7tf4zo2cv61rh75vg392g",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":2112507768413970,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10006_solr":250.0,
   [junit4]   2>       "127.0.0.1:10007_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10006_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10007_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "_enqueue_time_":2112508850680670}}
   [junit4]   2> 1767752 DEBUG (AutoscalingActionExecutor-5429-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 297 ms for event 
id=781508b1f1f12T5knc7tf4zo2cv61rh75vg392g
   [junit4]   2> 1767761 DEBUG (ScheduledTrigger-5428-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767769 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"78150c9f4daecT5knc7tf4zo2cv61rh75vg392j",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":2112508822608620,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10006_solr":250.0,
   [junit4]   2>       "127.0.0.1:10007_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10006_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10007_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}}}}
   [junit4]   2> 1767782 DEBUG (ScheduledTrigger-5428-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767790 DEBUG (ScheduledTrigger-5428-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"78150c9f4daecT5knc7tf4zo2cv61rh75vg392l",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":2112508822608620,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10006_solr":250.0,
   [junit4]   2>       "127.0.0.1:10007_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10006_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10007_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}}}}
   [junit4]   2> 1767792 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
   [junit4]   2> 1767792 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1767794 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1767795 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1767795 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1767795 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:              2
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:     0
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:              0
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:     [collection1]
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:   1
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:   1
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:          2
   [junit4]   2> 1767796 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * collection1            2
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - active           2
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - CREATE             1
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - autoscaling        2
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - update             4
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1767797 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: search_rate_trigger
   [junit4]   2> 1767798 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - IGNORED         2
   [junit4]   2> 1767798 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - STARTED         1
   [junit4]   2> 1767798 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##          - SUCCEEDED       1
   [junit4]   2> 1767803 DEBUG (ScheduledTrigger-5428-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767814 DEBUG (ScheduledTrigger-5428-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"78150c9f4daecT5knc7tf4zo2cv61rh75vg392n",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":2112508822608620,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10006_solr":250.0,
   [junit4]   2>       "127.0.0.1:10007_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10006_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10007_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}}}}
   [junit4]   2> 1767824 DEBUG (ScheduledTrigger-5428-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767839 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 1767839 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 1767839 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 1767839 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1767839 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1767840 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767842 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1767842 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1767844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1767844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 6
   [junit4]   2> 1767844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 1767844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1767845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1767845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1767845 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767865 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767891 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767911 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767934 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1767945 DEBUG (simCloudManagerPool-5430-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1767945 DEBUG (simCloudManagerPool-5430-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 1767948 DEBUG (simCloudManagerPool-5430-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767948 DEBUG (simCloudManagerPool-5430-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10007_solr]
   [junit4]   2> 1767948 DEBUG (simCloudManagerPool-5430-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1767948 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 1767948 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1767948 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 1767948 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1767948 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1767954 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1767973 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1767973 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1767973 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10008_solr at time 
2112520426141870
   [junit4]   2> 1767974 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1767994 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1767994 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1767995 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768014 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1768014 DEBUG (ScheduledTrigger-5431-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768015 DEBUG (ScheduledTrigger-5431-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768037 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1768037 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768037 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10008_solr] added at times 
[2112520426141870], now=2112523601057170
   [junit4]   2> 1768037 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"781537d94b8aeT5knc7tf4zo2cv61rh75vg392p",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":2112520426141870,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112520426141870],
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1768037 DEBUG (AutoscalingActionExecutor-5432-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"781537d94b8aeT5knc7tf4zo2cv61rh75vg392p",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":2112520426141870,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112520426141870],
   [junit4]   2>     "_enqueue_time_":2112523601072270,
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1768041 DEBUG (AutoscalingActionExecutor-5432-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 215 ms for event 
id=781537d94b8aeT5knc7tf4zo2cv61rh75vg392p
   [junit4]   2> 1768042 DEBUG (simCloudManagerPool-5430-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768042 DEBUG (simCloudManagerPool-5430-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 1768042 DEBUG (simCloudManagerPool-5430-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10008_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1768042 DEBUG (simCloudManagerPool-5430-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10008_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1768042 DEBUG (simCloudManagerPool-5430-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1768042 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 1768042 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 1768057 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768057 DEBUG (ScheduledTrigger-5431-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1768057 DEBUG (ScheduledTrigger-5431-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768078 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768078 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1768078 DEBUG (ScheduledTrigger-5431-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:                3
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:       3
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:                0
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:       []
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:     0
   [junit4]   2> 1768082 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:     0
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:            0
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##            - autoscaling        2
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##            - update             2
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##            - STARTED         1
   [junit4]   2> 1768083 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##            - SUCCEEDED       1
   [junit4]   2> 1768104 DEBUG (ScheduledTrigger-5431-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768104 DEBUG (ScheduledTrigger-5431-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1768106 DEBUG (ScheduledTrigger-5431-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768115 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger
   [junit4]   2> 1768116 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 1768116 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 1768116 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1768116 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1768116 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768118 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2>        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2>        at 
java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1768118 DEBUG 
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1768119 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1768119 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 10
   [junit4]   2> 1768119 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768119 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1768119 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
10
   [junit4]   2> 1768120 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768120 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1768120 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768140 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768165 DEBUG (ScheduledTrigger-5434-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768185 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768213 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768220 DEBUG (simCloudManagerPool-5433-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768220 DEBUG (simCloudManagerPool-5433-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 1768222 DEBUG (simCloudManagerPool-5433-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768223 DEBUG (simCloudManagerPool-5433-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
11
   [junit4]   2> 1768223 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1768225 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768225 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 1768225 DEBUG (ScheduledTrigger-5434-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768225 DEBUG (ScheduledTrigger-5434-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr
   [junit4]   2> 1768233 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768233 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr
   [junit4]   2> 1768250 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768253 DEBUG (ScheduledTrigger-5434-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768270 DEBUG (ScheduledTrigger-5434-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768275 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768290 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768291 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost 
nodes: [127.0.0.1:10006_solr]
   [junit4]   2> 1768291 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"781566ccb5becT5knc7tf4zo2cv61rh75vg392s",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2112533029411820,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112533029411820],
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 1768291 DEBUG (AutoscalingActionExecutor-5435-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"781566ccb5becT5knc7tf4zo2cv61rh75vg392s",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2112533029411820,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112533029411820],
   [junit4]   2>     "_enqueue_time_":2112536295347670,
   [junit4]   2>     "nodeNames":["127.0.0.1:10006_solr"]}}
   [junit4]   2> 1768292 DEBUG (AutoscalingActionExecutor-5435-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 19 ms for event 
id=781566ccb5becT5knc7tf4zo2cv61rh75vg392s
   [junit4]   2> 1768292 DEBUG (simCloudManagerPool-5433-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768292 DEBUG (simCloudManagerPool-5433-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 1768292 DEBUG (simCloudManagerPool-5433-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr]
   [junit4]   2> 1768292 DEBUG (simCloudManagerPool-5433-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10008_solr]
   [junit4]   2> 1768292 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
12
   [junit4]   2> 1768292 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768292 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 1768302 DEBUG (ScheduledTrigger-5434-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768311 DEBUG (ScheduledTrigger-5434-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768322 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768332 DEBUG (ScheduledTrigger-5434-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768332 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:         1
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:        1
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:         1
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             127.0.0.1:10006_solr
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:        []
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:      0
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:      0
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:             0
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - autoscaling        2
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - update             2
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - STARTED         1
   [junit4]   2> 1768333 INFO  
(TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[EA8383113195D55C]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - SUCCEEDED       1
   [junit4]   2> 1768343 DEBUG (ScheduledTrigger-5434-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768353 DEBUG (ScheduledTrigger-5434-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with 
currently live nodes: 1
   [junit4]   2> 1768364 DEBUG (ScheduledTrigger-5434-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 1
   [junit4]   2> 1768364 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testCooldown
   [junit4]   2> 1768364 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
13
   [junit4]   2> 1768364 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
13
   [junit4]   2> 1768364 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1768364 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768364 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 1768364 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10006_solr
   [junit4]   2> 1768364 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 1768367 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1768367 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 14
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
14
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768369 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 1768369 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768390 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768414 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768434 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768454 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768469 DEBUG (simCloudManagerPool-5436-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768469 DEBUG (simCloudManagerPool-5436-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
15
   [junit4]   2> 1768472 DEBUG (simCloudManagerPool-5436-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768472 DEBUG (simCloudManagerPool-5436-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1768472 DEBUG (simCloudManagerPool-5436-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
15
   [junit4]   2> 1768472 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1768472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 1768473 DEBUG (simCloudManagerPool-5436-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768473 DEBUG (simCloudManagerPool-5436-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
16
   [junit4]   2> 1768473 DEBUG (simCloudManagerPool-5436-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768473 DEBUG (simCloudManagerPool-5436-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 1768473 DEBUG (simCloudManagerPool-5436-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10008_solr]
   [junit4]   2> 1768473 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
16
   [junit4]   2> 1768473 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
   [junit4]   2> 1768476 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768476 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768476 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768477 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10010_solr at time 
2112545586502920
   [junit4]   2> 1768503 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768509 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768509 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768509 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10010_solr] added at times 
[2112545586502920], now=2112547216555370
   [junit4]   2> 1768509 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"7815959415d08T5knc7tf4zo2cv61rh75vg392v",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112545586502920,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112545586502920],
   [junit4]   2>     "nodeNames":["127.0.0.1:10010_solr"]}}
   [junit4]   2> 1768510 DEBUG (AutoscalingActionExecutor-5438-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"7815959415d08T5knc7tf4zo2cv61rh75vg392v",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112545586502920,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112545586502920],
   [junit4]   2>     "_enqueue_time_":2112547222803270,
   [junit4]   2>     "nodeNames":["127.0.0.1:10010_solr"]}}
   [junit4]   2> 1768510 DEBUG (AutoscalingActionExecutor-5438-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 17 ms for event 
id=7815959415d08T5knc7tf4zo2cv61rh75vg392v
   [junit4]   2> 1768523 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768530 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768530 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768543 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768553 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768553 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768553 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10011_solr at time 
2112549410036420
   [junit4]   2> 1768563 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768573 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768573 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768573 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10011_solr] added at times 
[2112549410036420], now=2112550423441470
   [junit4]   2> 1768573 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815a3d27dac4T5knc7tf4zo2cv61rh75vg392y",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112549410036420,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112549410036420],
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1768584 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768596 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768597 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768597 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10011_solr] added at times 
[2112549410036420], now=2112551592186320
   [junit4]   2> 1768597 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815a3d27dac4T5knc7tf4zo2cv61rh75vg3930",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112549410036420,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112549410036420],
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1768604 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768617 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768618 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768618 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10011_solr] added at times 
[2112549410036420], now=2112552641539270
   [junit4]   2> 1768618 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"7815a3d27dac4T5knc7tf4zo2cv61rh75vg3932",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112549410036420,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112549410036420],
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1768618 DEBUG (AutoscalingActionExecutor-5438-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"7815a3d27dac4T5knc7tf4zo2cv61rh75vg3932",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":2112549410036420,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112549410036420],
   [junit4]   2>     "_enqueue_time_":2112552647761920,
   [junit4]   2>     "nodeNames":["127.0.0.1:10011_solr"]}}
   [junit4]   2> 1768619 DEBUG (AutoscalingActionExecutor-5438-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 32 ms for event 
id=7815a3d27dac4T5knc7tf4zo2cv61rh75vg3932
   [junit4]   2> 1768624 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768638 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768639 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768644 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768659 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testCooldown
   [junit4]   2> 1768659 DEBUG (ScheduledTrigger-5437-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:                4
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:       4
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:                0
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:       []
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:     0
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:     0
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:            0
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - autoscaling        2
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - update             6
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_cooldown_trigger
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - IGNORED         2
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - STARTED         2
   [junit4]   2> 1768659 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[EA8383113195D55C]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##            - SUCCEEDED       2
   [junit4]   2> 1768666 DEBUG (ScheduledTrigger-5437-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768681 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 1768681 DEBUG (ScheduledTrigger-5437-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 1768690 DEBUG (ScheduledTrigger-5437-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 1768698 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[EA8383113195D55C]) [   
 ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
   [junit4]   2> 1768698 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[EA8383113195D55C]) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 17
   [junit4]   2> 1768698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
17
   [junit4]   2> 1768698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1768699 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768699 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17
   [junit4]   2> 1768699 INFO  
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[EA8383113195D55C]) [   
 ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and 
clearing object cache...
   [junit4]   2> 1768701 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1768701 DEBUG 
(TEST-TestTriggerIntegration.testTriggerThrottling-seed#[EA8383113195D55C]) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 1768702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1768702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 18
   [junit4]   2> 1768702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion -1
   [junit4]   2> 1768702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
18
   [junit4]   2> 1768703 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1768703 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18
   [junit4]   2> 1768703 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768723 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768748 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768768 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768790 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 1768803 DEBUG (simCloudManagerPool-5439-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768803 DEBUG (simCloudManagerPool-5439-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
19
   [junit4]   2> 1768806 DEBUG (simCloudManagerPool-5439-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768806 DEBUG (simCloudManagerPool-5439-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768806 DEBUG (simCloudManagerPool-5439-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1768806 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
19
   [junit4]   2> 1768806 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1768806 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19
   [junit4]   2> 1768806 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768806 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
20
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10009_solr, 
127.0.0.1:10011_solr]
   [junit4]   2> 1768807 DEBUG (simCloudManagerPool-5439-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}],
 enabled=true}
   [junit4]   2> 1768807 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 
20
   [junit4]   2> 1768807 INFO  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1768807 DEBUG (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20
   [junit4]   2> 1768808 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768808 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768808 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 
2112562150869570
   [junit4]   2> 1768808 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112562150869570], now=2112562152573620
   [junit4]   2> 1768808 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"7815d34918a42T5knc7tf4zo2cv61rh75vg3935",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":2112562150869570,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112562150869570],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768808 DEBUG (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"7815d34918a42T5knc7tf4zo2cv61rh75vg3935",
   [junit4]   2>   "source":"node_added_trigger2",
   [junit4]   2>   "eventTime":2112562150869570,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112562150869570],
   [junit4]   2>     "_enqueue_time_":2112562160303920,
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768808 INFO  (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2
   [junit4]   2> 1768809 DEBUG (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 23 ms for event 
id=7815d34918a42T5knc7tf4zo2cv61rh75vg3935
   [junit4]   2> 1768811 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768831 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10012_solr at time 
2112563452240370
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112563452240370], now=2112563455838220
   [junit4]   2> 1768834 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg3938",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768852 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112563452240370], now=2112564579701120
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768856 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg393a",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768872 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768877 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768877 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768878 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768878 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768878 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112563452240370], now=2112565654366970
   [junit4]   2> 1768878 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg393c",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768892 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768897 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768897 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768899 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768899 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768899 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112563452240370], now=2112566707921070
   [junit4]   2> 1768899 DEBUG (ScheduledTrigger-5440-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg393e",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768913 DEBUG (ScheduledTrigger-5440-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 1768917 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
   [junit4]   2> 1768917 DEBUG (ScheduledTrigger-5440-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768920 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 1768920 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1768920 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10012_solr] added at times 
[2112563452240370], now=2112567757938920
   [junit4]   2> 1768920 DEBUG (ScheduledTrigger-5440-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg393g",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768921 DEBUG (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"7815d8222e1f2T5knc7tf4zo2cv61rh75vg393g",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":2112563452240370,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112563452240370],
   [junit4]   2>     "_enqueue_time_":2112567764351370,
   [junit4]   2>     "nodeNames":["127.0.0.1:10012_solr"]}}
   [junit4]   2> 1768921 INFO  (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration last action at 2112562191973120 time = 
2112567805894020
   [junit4]   2> 1768921 INFO  (AutoscalingActionExecutor-5441-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1
  

[...truncated too long message...]

nodes: [127.0.0.1:10021_solr]
   [junit4]   2> 1771083 DEBUG (ScheduledTrigger-5470-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"7817752e671b4T5knc7tf4zo2cv61rh75vg394q",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2112674328900020,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112674328900020],
   [junit4]   2>     "nodeNames":["127.0.0.1:10021_solr"]}}
   [junit4]   2> 1771083 DEBUG (AutoscalingActionExecutor-5471-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"7817752e671b4T5knc7tf4zo2cv61rh75vg394q",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2112674328900020,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2112674328900020],
   [junit4]   2>     "_enqueue_time_":2112675923902020,
   [junit4]   2>     "nodeNames":["127.0.0.1:10021_solr"]}}
   [junit4]   2> 1771083 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 1771083 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1771083 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS 
############
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
#############################################
   [junit4]   2> 
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:             2
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:    2
   [junit4]   2> 1771084 DEBUG (AutoscalingActionExecutor-5471-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 26 ms for event 
id=7817752e671b4T5knc7tf4zo2cv61rh75vg394q
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:             1
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         127.0.0.1:10021_solr
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:    []
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:  0
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:  0
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:         0
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts 
##########
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - autoscaling        2
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - update             2
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_restore_trigger
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - STARTED         1
   [junit4]   2> 1771084 INFO  
(TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[EA8383113195D55C])
 [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##         - SUCCEEDED       1
   [junit4]   2> 1771084 WARN  (Simulated OverseerAutoScalingTriggerThread) [   
 ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 1771090 DEBUG 
(SUITE-TestTriggerIntegration-seed#[EA8383113195D55C]-worker) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_EA8383113195D55C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=1990, maxMBSortInHeap=6.6282052008461045, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@30d5a63),
 locale=sr-ME, timezone=America/St_Lucia
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 
(64-bit)/cpus=3,threads=1,free=196755080,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldSimilarity, 
TestLeaderElectionZkExpiry, TestSuggestSpellingConverter, TestZkChroot, 
TestFaceting, SearchHandlerTest, DistributedFacetPivotWhiteBoxTest, 
TestIntervalFaceting, TestSmileRequest, TestSegmentSorting, SolrCLIZkUtilsTest, 
TestFieldCacheSortRandom, SampleTest, RemoteQueryErrorTest, 
TestLegacyNumericRangeQueryBuilder, CloneFieldUpdateProcessorFactoryTest, 
TestFastWriter, StressHdfsTest, HdfsThreadLeakTest, TestUninvertingReader, 
TestCSVLoader, TestClusterStateProvider, CoreAdminHandlerTest, 
TestFileDictionaryLookup, DirectoryFactoryTest, IndexSchemaRuntimeFieldTest, 
TestPayloadCheckQParserPlugin, JvmMetricsTest, TestStressVersions, 
TestCustomSort, DocumentBuilderTest, TestJsonFacetRefinement, ZkSolrClientTest, 
StatsReloadRaceTest, TermVectorComponentDistributedTest, TestDefaultStatsCache, 
DocValuesMissingTest, SchemaVersionSpecificBehaviorTest, 
TestUnifiedSolrHighlighter, TestRTGBase, ShardRoutingCustomTest, 
FieldMutatingUpdateProcessorTest, PeerSyncReplicationTest, FacetPivotSmallTest, 
TestNumericTerms32, JsonLoaderTest, ZkStateReaderTest, 
HdfsUnloadDistributedZkTest, CdcrRequestHandlerTest, PolyFieldTest, 
TestNoOpRegenerator, ImplicitSnitchTest, TestFastOutputStream, TestSQLHandler, 
BooleanFieldTest, ExecutePlanActionTest, TestGroupingSearch, 
UUIDUpdateProcessorFallbackTest, CloudExitableDirectoryReaderTest, 
TestLegacyTerms, TestInPlaceUpdatesDistrib, TestSort, DistributedQueueTest, 
FileUtilsTest, FullSolrCloudDistribCmdsTest, SimpleFacetsTest, 
TestDeleteCollectionOnDownNodes, OverseerModifyCollectionTest, 
TestObjectReleaseTracker, TestPrepRecovery, SpatialHeatmapFacetsTest, 
TestDelegationWithHadoopAuth, TestStressInPlaceUpdates, 
TestHalfAndHalfDocValues, TestCollectionAPIs, 
DistributedFacetPivotSmallAdvancedTest, SolrXmlInZkTest, TestSolrXml, 
DistributedMLTComponentTest, TestBinaryResponseWriter, MigrateRouteKeyTest, 
TestBackupRepositoryFactory, ActionThrottleTest, TestFoldingMultitermQuery, 
TestWordDelimiterFilterFactory, BasicFunctionalityTest, V2StandaloneTest, 
QueryEqualityTest, ManagedSchemaRoundRobinCloudTest, SolrSlf4jReporterTest, 
TermVectorComponentTest, TestLocalFSCloudBackupRestore, 
QueryElevationComponentTest, TestReload, LeaderFailureAfterFreshStartTest, 
DataDrivenBlockJoinTest, TestSolrCoreSnapshots, TestFieldCacheReopen, 
TestDocSet, SolrJmxReporterCloudTest, TestSchemaNameResource, 
LeaderElectionIntegrationTest, TestJsonRequest, TemplateUpdateProcessorTest, 
JavabinLoaderTest, OverriddenZkACLAndCredentialsProvidersTest, 
ChaosMonkeyNothingIsSafeWithPullReplicasTest, 
UpdateRequestProcessorFactoryTest, BadComponentTest, PreAnalyzedFieldTest, 
TestDownShardTolerantSearch, SecurityConfHandlerTest, 
CoreAdminCreateDiscoverTest, SolrCoreMetricManagerTest, 
VMParamsZkACLAndCredentialsProvidersTest, TestCharFilters, 
SearchRateTriggerTest, TestLMJelinekMercerSimilarityFactory, DOMUtilTest, 
TestFunctionQuery, TestIndexSearcher, TestRuleBasedAuthorizationPlugin, 
TestFastLRUCache, FullHLLTest, SearchHandlerTest, CacheHeaderTest, 
MetricsHandlerTest, SolrCoreTest, AnalysisAfterCoreReloadTest, 
TestSystemCollAutoCreate, TestExactSharedStatsCache, SolrGangliaReporterTest, 
TimeRoutedAliasUpdateProcessorTest, TestNodeAddedTrigger, 
HttpTriggerListenerTest, LeaderElectionContextKeyTest, UtilsToolTest, 
TestRemoteStreaming, TestMergePolicyConfig, SOLR749Test, 
CdcrReplicationDistributedZkTest, TestReplicaProperties, 
SpellingQueryConverterTest, TestConfigSets, BlockJoinFacetDistribTest, 
TestSolrQueryParser, CollectionStateFormat2Test, HttpSolrCallGetCoreTest, 
AssignTest, TestFieldTypeCollectionResource, TestCloudSearcherWarming, 
MetricUtilsTest, TestSubQueryTransformerDistrib, DocValuesTest, 
TestXmlQParserPlugin, DeleteStatusTest, TestExtendedDismaxParser, 
PreAnalyzedUpdateProcessorTest, TestFilteredDocIdSet, UpdateLogTest, 
TestTriggerIntegration]
   [junit4] Completed [290/765 (1!)] on J1 in 3.60s, 11 tests, 1 error <<< 
FAILURES!

[...truncated 49684 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to