Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/315/

5 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 
seconds
        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:45E424CF2DA2D387]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
        at 
org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
        at 
org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:913)
        at 
org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:428)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:938BE997A362DE4C]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown

Error Message:
[CapturedEvent{timestamp=462370991008577, stage=SUCCEEDED, actionName='null', 
event={   "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",   
"source":"node_added_cooldown_trigger",   "eventTime":462368175425427,   
"eventType":"NODEADDED",   "properties":{     "eventTimes":[462368175425427],   
  "_enqueue_time_":462369981194277,     "nodeNames":["127.0.0.1:10005_solr"]}}, 
context={}, config={   "trigger":"node_added_cooldown_trigger",   
"afterAction":[],   "stage":[     "FAILED",     "SUCCEEDED",     "IGNORED"],   
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
   "beforeAction":[]}, message='null'}]

Stack Trace:
java.lang.AssertionError: [CapturedEvent{timestamp=462370991008577, 
stage=SUCCEEDED, actionName='null', event={
  "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
  "source":"node_added_cooldown_trigger",
  "eventTime":462368175425427,
  "eventType":"NODEADDED",
  "properties":{
    "eventTimes":[462368175425427],
    "_enqueue_time_":462369981194277,
    "nodeNames":["127.0.0.1:10005_solr"]}}, context={}, config={
  "trigger":"node_added_cooldown_trigger",
  "afterAction":[],
  "stage":[
    "FAILED",
    "SUCCEEDED",
    "IGNORED"],
  
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
  "beforeAction":[]}, message='null'}]
        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:FF7D9AFA120E0DF1]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1100)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:2303600ED1C8E5DA]:0)
        at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
        at java.util.ArrayList$Itr.next(ArrayList.java:859)
        at java.util.AbstractCollection.toString(AbstractCollection.java:461)
        at java.lang.String.valueOf(String.java:2994)
        at java.lang.StringBuilder.append(StringBuilder.java:131)
        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:962)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig

Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{   
"cluster-preferences":[{"maximize":"freedisk"}],   
"triggers":{".auto_add_replicas":{       "name":".auto_add_replicas",       
"event":"nodeLost",       "waitFor":30,       "actions":[         {           
"name":"auto_add_replicas_plan",           
"class":"solr.AutoAddReplicasPlanAction"},         {           
"name":"execute_plan",           "class":"solr.ExecutePlanAction"}],       
"enabled":true}},   "listeners":{".auto_add_replicas.system":{       
"trigger":".auto_add_replicas",       "afterAction":[],       "stage":[         
"STARTED",         "ABORTED",         "SUCCEEDED",         "FAILED",         
"BEFORE_ACTION",         "AFTER_ACTION",         "IGNORED"],       
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",       
"beforeAction":[]}},   "properties":{}}> but was: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{   
"cluster-preferences":[{"maximize":"freedisk"}],   
"triggers":{".auto_add_replicas":{       "name":".auto_add_replicas",       
"event":"nodeLost",       "waitFor":30,       "actions":[         {           
"name":"auto_add_replicas_plan",           
"class":"solr.AutoAddReplicasPlanAction"},         {           
"name":"execute_plan",           "class":"solr.ExecutePlanAction"}],       
"enabled":true}},   "listeners":{".auto_add_replicas.system":{       
"trigger":".auto_add_replicas",       "afterAction":[],       "stage":[         
"STARTED",         "ABORTED",         "SUCCEEDED",         "FAILED",         
"BEFORE_ACTION",         "AFTER_ACTION",         "IGNORED"],       
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",       
"beforeAction":[]}},   "properties":{}}>

Stack Trace:
java.lang.AssertionError: expected: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
  "cluster-preferences":[{"maximize":"freedisk"}],
  "triggers":{".auto_add_replicas":{
      "name":".auto_add_replicas",
      "event":"nodeLost",
      "waitFor":30,
      "actions":[
        {
          "name":"auto_add_replicas_plan",
          "class":"solr.AutoAddReplicasPlanAction"},
        {
          "name":"execute_plan",
          "class":"solr.ExecutePlanAction"}],
      "enabled":true}},
  "listeners":{".auto_add_replicas.system":{
      "trigger":".auto_add_replicas",
      "afterAction":[],
      "stage":[
        "STARTED",
        "ABORTED",
        "SUCCEEDED",
        "FAILED",
        "BEFORE_ACTION",
        "AFTER_ACTION",
        "IGNORED"],
      "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
      "beforeAction":[]}},
  "properties":{}}> but was: 
org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
  "cluster-preferences":[{"maximize":"freedisk"}],
  "triggers":{".auto_add_replicas":{
      "name":".auto_add_replicas",
      "event":"nodeLost",
      "waitFor":30,
      "actions":[
        {
          "name":"auto_add_replicas_plan",
          "class":"solr.AutoAddReplicasPlanAction"},
        {
          "name":"execute_plan",
          "class":"solr.ExecutePlanAction"}],
      "enabled":true}},
  "listeners":{".auto_add_replicas.system":{
      "trigger":".auto_add_replicas",
      "afterAction":[],
      "stage":[
        "STARTED",
        "ABORTED",
        "SUCCEEDED",
        "FAILED",
        "BEFORE_ACTION",
        "AFTER_ACTION",
        "IGNORED"],
      "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
      "beforeAction":[]}},
  "properties":{}}>
        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:F14BF4B67BF288E4]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:147)
        at 
org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12013 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_CEC3F71E6CA47803-001/init-core-data-001
   [junit4]   2> 568659 WARN  
(SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 568692 INFO  
(SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 568693 INFO  
(SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 568693 INFO  
(SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 568785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 568785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 0
   [junit4]   2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
   [junit4]   2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 568890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 568890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 568996 DEBUG (ScheduledTrigger-1073-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569078 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 569078 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
1
   [junit4]   2> 569079 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 569080 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 569100 DEBUG (ScheduledTrigger-1073-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569151 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 569151 WARN  (ScheduledTrigger-1073-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger NodeLostTrigger ran but was already closed
   [junit4]   2> 569151 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 569151 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 2
   [junit4]   2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 569152 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569188 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569222 DEBUG (ScheduledTrigger-1076-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569262 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569282 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569371 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569408 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569415 DEBUG (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 569415 DEBUG (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
3
   [junit4]   2> 569416 DEBUG (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 569434 INFO  (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 569434 DEBUG (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 
127.0.0.1:10000_solr]
   [junit4]   2> 569434 DEBUG (simCloudManagerPool-1075-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}],
 enabled=true}
   [junit4]   2> 569434 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 569434 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 569435 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 569444 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 
462262255869127
   [junit4]   2> 569469 DEBUG (ScheduledTrigger-1076-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing 
registered processor for nodes: [127.0.0.1:10002_solr] added at times 
[462262255869127], now=462263447164177
   [junit4]   2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 569470 DEBUG (AutoscalingActionExecutor-1077-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "_enqueue_time_":462263461237677,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 569471 INFO  (AutoscalingActionExecutor-1077-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "_enqueue_time_":462263461237677,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"]}}
   [junit4]   2> 569471 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 569471 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 569471 DEBUG 
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 569472 WARN  (AutoscalingActionExecutor-1077-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Exception executing actions
   [junit4]   2> org.apache.lucene.store.AlreadyClosedException: 
ScheduledTrigger node_added_trigger1 has been closed.
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:494)
   [junit4]   2>        at 
org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:306)
   [junit4]   2>        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2>        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 569472 DEBUG (AutoscalingActionExecutor-1077-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 70 ms for event 
id=1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 3
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569472 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}],
 enabled=true}
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion -1
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 
127.0.0.1:10001_solr
   [junit4]   2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 569473 DEBUG (ScheduledTrigger-1079-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569473 DEBUG (ScheduledTrigger-1079-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 569473 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "_dequeue_time_":462263642306527,
   [junit4]   2>     "_enqueue_time_":462263461237677,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 569474 DEBUG (AutoscalingActionExecutor-1080-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "_dequeue_time_":462263642306527,
   [junit4]   2>     "_enqueue_time_":462263461237677,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 569474 INFO  (AutoscalingActionExecutor-1080-thread-1) [    ] 
o.a.s.c.a.s.TestTriggerIntegration -- event: {
   [junit4]   2>   "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":462262255869127,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462262255869127],
   [junit4]   2>     "_dequeue_time_":462263642306527,
   [junit4]   2>     "_enqueue_time_":462263461237677,
   [junit4]   2>     "nodeNames":["127.0.0.1:10002_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:               2
   [junit4]   2> 569582 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:      2
   [junit4]   2> 569583 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:               1
   [junit4]   2> 569583 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           127.0.0.1:10001_solr
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:      []
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:    0
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:    0
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:           0
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - autoscaling        1
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - update             2
   [junit4]   2> 569584 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 569585 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
   [junit4]   2> 569585 INFO  
(TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - STARTED         2
   [junit4]   2> 569595 DEBUG (AutoscalingActionExecutor-1080-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 6091 ms for event 
id=1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7
   [junit4]   2> 569601 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
   [junit4]   2> 569601 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 4
   [junit4]   2> 569601 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 
127.0.0.1:10001_solr
   [junit4]   2> 569601 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 569601 DEBUG 
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed 
explicitly
   [junit4]   2> 569619 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 5
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion -1
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 569631 DEBUG (ScheduledTrigger-1082-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569655 DEBUG (ScheduledTrigger-1082-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569704 DEBUG (ScheduledTrigger-1082-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
6
   [junit4]   2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 569722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 569722 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 569722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 
462278160743427
   [junit4]   2> 569764 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing 
registered processor for nodes: [127.0.0.1:10003_solr] added at times 
[462278160743427], now=462279638884277
   [junit4]   2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":462278160743427,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462278160743427],
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 569794 DEBUG (AutoscalingActionExecutor-1083-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb",
   [junit4]   2>   "source":"node_added_trigger",
   [junit4]   2>   "eventTime":462278160743427,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462278160743427],
   [junit4]   2>     "_enqueue_time_":462279646129827,
   [junit4]   2>     "nodeNames":["127.0.0.1:10003_solr"]}}
   [junit4]   2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
7
   [junit4]   2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
   [junit4]   2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated 
with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 569795 DEBUG (AutoscalingActionExecutor-1083-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 78 ms for event 
id=1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb
   [junit4]   2> 569815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 569815 DEBUG (ScheduledTrigger-1082-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 569829 DEBUG (ScheduledTrigger-1082-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569829 DEBUG (ScheduledTrigger-1082-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569835 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:         3
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:        3
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:         0
   [junit4]   2> 569835 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:        []
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:      0
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:      0
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:             0
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - autoscaling        2
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - update             2
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts 
###########
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - STARTED         1
   [junit4]   2> 569836 INFO  
(TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [    
] o.a.s.c.a.s.SimSolrCloudTestCase ##             - SUCCEEDED       1
   [junit4]   2> 569851 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569851 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 569920 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
   [junit4]   2> 569920 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
8
   [junit4]   2> 569921 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 569921 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 569921 DEBUG 
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 9
   [junit4]   2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion -1
   [junit4]   2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 569943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 569943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 570012 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570032 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570057 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570093 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570121 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570149 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570178 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570195 DEBUG (simCloudManagerPool-1084-thread-4) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 570195 DEBUG (simCloudManagerPool-1084-thread-4) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
10
   [junit4]   2> 570232 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570252 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570267 DEBUG (simCloudManagerPool-1084-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 570287 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570323 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570340 DEBUG (simCloudManagerPool-1084-thread-5) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 570340 DEBUG (simCloudManagerPool-1084-thread-5) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
11
   [junit4]   2> 570401 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570403 DEBUG (simCloudManagerPool-1084-thread-5) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 570421 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570485 DEBUG (ScheduledTrigger-1085-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570506 DEBUG (ScheduledTrigger-1085-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570619 DEBUG (ScheduledTrigger-1085-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570647 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570684 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570685 DEBUG (ScheduledTrigger-1085-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":462314364637777,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10000_solr":250.0,
   [junit4]   2>       "127.0.0.1:10002_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10000_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10002_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}}}}
   [junit4]   2> 570801 DEBUG (AutoscalingActionExecutor-1086-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":462314364637777,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{
   [junit4]   2>       "127.0.0.1:10000_solr":250.0,
   [junit4]   2>       "127.0.0.1:10002_solr":250.0},
   [junit4]   2>     "replica":[
   [junit4]   2>       "{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10000_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}",
   [junit4]   2>       "{\"core_node1\":{\n    
\"core\":\"collection1_shard1_replica_n1\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10002_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":500.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":500.0}},
   [junit4]   2>     "_enqueue_time_":462324241524177}}
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 570856 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:               2
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:      0
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:               0
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:      [collection1]
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:    1
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:    1
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:           2
   [junit4]   2> 570857 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * collection1             2
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - active           2
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - CREATE             1
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - autoscaling        2
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - update             1
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: search_rate_trigger
   [junit4]   2> 570858 INFO  
(TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##           - STARTED         1
   [junit4]   2> 570948 DEBUG (AutoscalingActionExecutor-1086-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 12198 ms for event 
id=1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=TestTriggerIntegration -Dtests.method=testSearchRate 
-Dtests.seed=CEC3F71E6CA47803 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=ja-JP -Dtests.timezone=Europe/Vaduz -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 1.11s J2 | TestTriggerIntegration.testSearchRate <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The trigger did not 
fire at all
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:938BE997A362DE4C]:0)
   [junit4]    >        at 
org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
   [junit4]    >        at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 570984 DEBUG (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 570987 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting testCooldown
   [junit4]   2> 570987 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
12
   [junit4]   2> 570987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
   [junit4]   2> 571012 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing 
object cache...
   [junit4]   2> 571012 WARN  (ScheduledTrigger-1085-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers Ignoring autoscaling event {
   [junit4]   2>   "id":"1a47aa54d14a3Tdh05m3uxqd3e1aovccmuk6ldh",
   [junit4]   2>   "source":"search_rate_trigger",
   [junit4]   2>   "eventTime":462321642968227,
   [junit4]   2>   "eventType":"SEARCHRATE",
   [junit4]   2>   "properties":{
   [junit4]   2>     "node":{"127.0.0.1:10000_solr":250.0},
   [junit4]   2>     "replica":["{\"core_node2\":{\n    
\"core\":\"collection1_shard1_replica_n2\",\n    
\"INDEX.sizeInBytes\":123450000,\n    \"rate\":250.0,\n    
\"node_name\":\"127.0.0.1:10000_solr\",\n    \"state\":\"active\",\n    
\"type\":\"NRT\",\n    \"QUERY./select.requestTimes:1minRate\":250.0,\n    
\"shard\":\"shard1\",\n    \"collection\":\"collection1\"}}"],
   [junit4]   2>     "collection":{"collection1":250.0},
   [junit4]   2>     "shard":{"collection1":{"shard1":250.0}}}} because the 
source trigger: search_rate_trigger doesn't exist.
   [junit4]   2> 571107 WARN  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are 
closed, exiting.
   [junit4]   2> 571121 DEBUG 
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode 
version 13
   [junit4]   2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1
   [junit4]   2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
   [junit4]   2> 571162 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 571162 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
   [junit4]   2> 571162 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 571328 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 571331 DEBUG (simCloudManagerPool-1087-thread-1) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 571331 DEBUG (simCloudManagerPool-1087-thread-1) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
14
   [junit4]   2> 571348 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 2
   [junit4]   2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 571361 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
   [junit4]   2> 571361 INFO  (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 571361 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
   [junit4]   2> 571361 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571361 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [    ] 
o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [    ] 
o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 
15
   [junit4]   2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger 
instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, 
class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}],
 enabled=true}
   [junit4]   2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 
127.0.0.1:10002_solr]
   [junit4]   2> 571363 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
   [junit4]   2> 571363 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    
] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
   [junit4]   2> 571368 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 
462359080776977
   [junit4]   2> 571389 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10004_solr] added at times 
[462359080776977], now=462360854030827
   [junit4]   2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":462359080776977,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462359080776977],
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 571418 DEBUG (AutoscalingActionExecutor-1089-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":462359080776977,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462359080776977],
   [junit4]   2>     "_enqueue_time_":462360861686077,
   [junit4]   2>     "nodeNames":["127.0.0.1:10004_solr"]}}
   [junit4]   2> 571418 DEBUG (AutoscalingActionExecutor-1089-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 17 ms for event 
id=1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi
   [junit4]   2> 571439 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 571563 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 3
   [junit4]   2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [    ] 
o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 
462368175425427
   [junit4]   2> 571599 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing 
registered processor for nodes: [127.0.0.1:10005_solr] added at times 
[462368175425427], now=462369974245877
   [junit4]   2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [    ] 
o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":462368175425427,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462368175425427],
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 571617 DEBUG (AutoscalingActionExecutor-1089-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
   [junit4]   2>   "source":"node_added_cooldown_trigger",
   [junit4]   2>   "eventTime":462368175425427,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[462368175425427],
   [junit4]   2>     "_enqueue_time_":462369981194277,
   [junit4]   2>     "nodeNames":["127.0.0.1:10005_solr"]}}
   [junit4]   2> 571620 DEBUG (AutoscalingActionExecutor-1089-thread-1) [    ] 
o.a.s.c.a.ScheduledTriggers -- processing took 147 ms for event 
id=1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl
   [junit4]   2> 571636 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 571637 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571637 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with 
currently live nodes: 4
   [junit4]   2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
   [junit4]   2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [    ] 
o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
   [junit4]   2> 571672 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.SolrTestCaseJ4 ###Ending testCooldown
   [junit4]   2> 571672 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:         4
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:        4
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:         0
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:        []
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:      0
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:      0
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:             0
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##             - autoscaling        2
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ##             - update             4
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 571673 INFO  
(TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [    ] 
o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_cooldown_trigger
   [junit4]   2> 571673 INFO  (TEST-TestTriggerIntegratio

[...truncated too long message...]

]   2> 3577646 INFO  
(zkCallback-7765-thread-3-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (0)
   [junit4]   2> 3580143 WARN  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 3580143 INFO  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, 
but we have no versions - we can't sync in that case - we were active before, 
so become leader anyway
   [junit4]   2> 3580143 INFO  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2]  
CLOSING SolrCore org.apache.solr.core.SolrCore@54e775ec
   [junit4]   2> 3580144 INFO  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters 
for registry=solr.core..system.shard1.replica_n2, tag=1424455148
   [junit4]   2> 3580144 INFO  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter 
[org.apache.solr.metrics.reporters.SolrJmxReporter@25b32ffe: rootName = 
solr_42976, domain = solr.core..system.shard1.replica_n2, service url = null, 
agent id = null] for registry solr.core..system.shard1.replica_n2 / 
com.codahale.metrics.MetricRegistry@6aad0356
   [junit4]   2> 3580151 INFO  
(zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) 
[n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 
x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters 
for registry=solr.collection..system.shard1.leader, tag=1424455148
   [junit4]   2> 3580152 INFO  (jetty-closer-7747-thread-2) [    ] 
o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1acc79ee{/solr,null,UNAVAILABLE}
   [junit4]   2> 3580153 ERROR 
(SUITE-TestClusterStateProvider-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper 
server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3580153 INFO  
(SUITE-TestClusterStateProvider-seed#[CEC3F71E6CA47803]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:38210 38210
   [junit4]   2> 3580210 INFO  (Thread-8872) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:38210 38210
   [junit4]   2> 3580211 WARN  (Thread-8872) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/aliases.json
   [junit4]   2>        4       /solr/security.json
   [junit4]   2>        3       /solr/configs/.system
   [junit4]   2>        3       /solr/configs/.system/managed-schema
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        26      /solr/collections/.system/state.json
   [junit4]   2>        5       /solr/clusterstate.json
   [junit4]   2>        5       /solr/clusterprops.json
   [junit4]   2>        3       /solr/autoscaling.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        5       /solr/live_nodes
   [junit4]   2>        5       /solr/collections
   [junit4]   2>        2       /solr/overseer/queue
   [junit4]   2>        2       /solr/autoscaling/events/.auto_add_replicas
   [junit4]   2>        2       /solr/overseer/queue-work
   [junit4]   2>        2       /solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_CEC3F71E6CA47803-001
   [junit4]   2> Jan 15, 2018 10:08:24 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, 
docValues:{}, maxPointsInLeafNode=367, maxMBSortInHeap=7.668266411331945, 
sim=RandomSimilarity(queryNorm=true): {}, locale=sr-Latn-RS, 
timezone=America/Argentina/La_Rioja
   [junit4]   2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 
1.8.0_152 (64-bit)/cpus=4,threads=1,free=109349360,total=510656512
   [junit4]   2> NOTE: All tests run in this JVM: 
[TestSolrCloudWithSecureImpersonation, TestFieldCacheSortRandom, 
TestCSVResponseWriter, TestSolrQueryResponse, SuggesterWFSTTest, 
TestSolrCoreProperties, TestFieldCache, TestEmbeddedSolrServerConstructors, 
LeaderElectionTest, BadComponentTest, SolrMetricsIntegrationTest, 
TestCollectionsAPIViaSolrCloudCluster, HdfsThreadLeakTest, NodeLostTriggerTest, 
IndexSchemaTest, ClassificationUpdateProcessorIntegrationTest, 
SolrCoreCheckLockOnStartupTest, TestNestedDocsSort, TestPullReplica, 
TestRealTimeGet, CoreAdminRequestStatusTest, TestDistribIDF, 
TestTolerantUpdateProcessorRandomCloud, TestQuerySenderListener, 
TestTriggerIntegration, HttpTriggerListenerTest, TestHashQParserPlugin, 
TestPartialUpdateDeduplication, TestSolrFieldCacheBean, 
ShufflingReplicaListTransformerTest, BlockJoinFacetSimpleTest, 
DisMaxRequestHandlerTest, DocValuesNotIndexedTest, TestCloudSchemaless, 
RequestLoggingTest, TestNamedUpdateProcessors, TestReloadDeadlock, 
TestHttpShardHandlerFactory, SharedFSAutoReplicaFailoverTest, 
SolrTestCaseJ4Test, QueryElevationComponentTest, TestFileDictionaryLookup, 
TestFieldCollectionResource, TestLeaderInitiatedRecoveryThread, 
TestMaxTokenLenTokenizer, TestTrieFacet, TestCloudSearcherWarming, 
CoreSorterTest, TestSolrConfigHandlerConcurrent, TestSystemIdResolver, 
PKIAuthenticationIntegrationTest, HdfsBasicDistributedZkTest, 
TestGroupingSearch, RecoveryAfterSoftCommitTest, EchoParamsTest, 
ZkNodePropsTest, TestUniqueKeyFieldResource, TestGraphMLResponseWriter, 
SpellCheckCollatorWithCollapseTest, TestTrie, CdcrBootstrapTest, 
TestSolrCLIRunExample, TestFunctionQuery, TriLevelCompositeIdRoutingTest, 
FieldMutatingUpdateProcessorTest, HdfsDirectoryFactoryTest, 
TestConfigSetsAPIZkFailure, TestPolicyCloud, StatsComponentTest, 
CheckHdfsIndexTest, TestNoOpRegenerator, LargeFieldTest, 
DistributedFacetPivotSmallAdvancedTest, TestCoreDiscovery, 
DocExpirationUpdateProcessorFactoryTest, TestGenericDistributedQueue, 
DistributedFacetExistsSmallTest, TestSolrDeletionPolicy2, SuggestComponentTest, 
DeleteNodeTest, TestLegacyFieldCache, TestMinMaxOnMultiValuedField, 
HighlighterTest, TestLRUCache, BlockCacheTest, RequiredFieldsTest, 
TestJsonRequest, TestCopyFieldCollectionResource, TestMiniSolrCloudClusterSSL, 
TestShardHandlerFactory, TestStressLiveNodes, TestCoreContainer, 
TimeZoneUtilsTest, TestHashPartitioner, TestComplexPhraseLeadingWildcard, 
LoggingHandlerTest, TestHdfsUpdateLog, SortByFunctionTest, 
CoreAdminHandlerTest, DeleteStatusTest, TestExtendedDismaxParser, 
TestSegmentSorting, SSLMigrationTest, DistributedSpellCheckComponentTest, 
OverseerStatusTest, LeaderElectionContextKeyTest, MetricsHandlerTest, 
TestFiltering, TestSchemaSimilarityResource, TestFuzzyAnalyzedSuggestions, 
ConfigSetsAPITest, SolrJmxReporterCloudTest, DefaultValueUpdateProcessorTest, 
TestNumericRangeQuery32, TestTolerantSearch, 
OverseerCollectionConfigSetProcessorTest, AutoAddReplicasIntegrationTest, 
ChaosMonkeyNothingIsSafeTest, SubstringBytesRefFilterTest, 
TestRuleBasedAuthorizationPlugin, LeaderInitiatedRecoveryOnCommitTest, 
PreAnalyzedFieldManagedSchemaCloudTest, AtomicUpdateProcessorFactoryTest, 
TestDynamicLoading, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, 
UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, 
BasicZkTest, RecoveryZkTest, ZkSolrClientTest, TestZkChroot, 
TestRandomDVFaceting, TestDistributedSearch, DistributedTermsComponentTest, 
TestSolr4Spatial, PeerSyncTest, DirectUpdateHandlerTest, TestIndexSearcher, 
SoftAutoCommitTest, HighlighterConfigTest, UpdateParamsTest, 
AlternateDirectoryTest, CopyFieldTest, ResponseLogComponentTest, 
SolrIndexConfigTest, TestMergePolicyConfig, MultiTermTest, TestDocSet, 
TestBinaryField, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, 
ExternalFileFieldSortTest, TestPostingsSolrHighlighter, 
DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, 
TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, 
TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, 
TestLMJelinekMercerSimilarityFactory, TestFastWriter, ResourceLoaderTest, 
TestFastOutputStream, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, 
URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, 
PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, 
RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestUtils, 
SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, 
DistributedMLTComponentTest, TestHighlightDedupGrouping, 
TestSimpleTrackingShardHandler, TestEmbeddedSolrServerAdminHandler, 
TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, ActionThrottleTest, 
AliasIntegrationTest, AsyncCallRequestStatusResponseTest, 
ChaosMonkeySafeLeaderWithPullReplicasTest, CollectionTooManyReplicasTest, 
CollectionsAPISolrJTest, DeleteInactiveReplicaTest, 
DistribJoinFromCollectionTest, ForceLeaderTest, MoveReplicaHDFSTest, 
TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestClusterProperties, 
TestCollectionAPI, TestExclusionRuleCollectionAccess, 
TestLeaderElectionZkExpiry, TestLocalFSCloudBackupRestore, 
TestRandomRequestDistribution, TestSolrCloudWithDelegationTokens, 
TestSolrCloudWithKerberosAlt, TestTlogReplica, TestClusterStateProvider]
   [junit4] Completed [586/767 (3!)] on J2 in 12.74s, 2 tests, 1 failure <<< 
FAILURES!

[...truncated 48844 lines...]
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to