Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1456/
3 tests failed. FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState Error Message: The trigger did not fire at all Stack Trace: java.lang.AssertionError: The trigger did not fire at all at __randomizedtesting.SeedInfo.seed([CA9ABFF875BC0E0D:CAAC0B385CB1AA67]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState(TestTriggerIntegration.java:673) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test Error Message: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/29)={ "pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{ "range":"80000000-b332ffff", "state":"active", "replicas":{"core_node6":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/tlog", "core":"solrj_collection3_shard1_replica_n1", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard2":{ "range":"b3330000-e665ffff", "state":"active", "replicas":{"core_node7":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/tlog", "core":"solrj_collection3_shard2_replica_n2", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard3":{ "range":"e6660000-1998ffff", "state":"active", "replicas":{"core_node8":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/", "base_url":"http://127.0.0.1:39843/pzz/jk", "node_name":"127.0.0.1:39843_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/tlog", "core":"solrj_collection3_shard3_replica_n3", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard4":{ "range":"19990000-4ccbffff", "state":"active", "replicas":{"core_node9":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/tlog", "core":"solrj_collection3_shard4_replica_n4", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard5":{ "range":"4ccc0000-7fffffff", "state":"active", "replicas":{"core_node10":{ "core":"solrj_collection3_shard5_replica_n5", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "state":"down", "type":"NRT"}}}}, "router":{ "field":"myOwnField", "name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"true", "nrtReplicas":"1", "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:40479_pzz%2Fjk, 127.0.0.1:41869_pzz%2Fjk, 127.0.0.1:34972_pzz%2Fjk, 127.0.0.1:39031_pzz%2Fjk, 127.0.0.1:39843_pzz%2Fjk] Stack Trace: java.lang.AssertionError: Expected numSlices=5 numReplicas=1 but found DocCollection(solrj_collection3//collections/solrj_collection3/state.json/29)={ "pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{ "range":"80000000-b332ffff", "state":"active", "replicas":{"core_node6":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/tlog", "core":"solrj_collection3_shard1_replica_n1", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard2":{ "range":"b3330000-e665ffff", "state":"active", "replicas":{"core_node7":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/tlog", "core":"solrj_collection3_shard2_replica_n2", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard3":{ "range":"e6660000-1998ffff", "state":"active", "replicas":{"core_node8":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/", "base_url":"http://127.0.0.1:39843/pzz/jk", "node_name":"127.0.0.1:39843_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/tlog", "core":"solrj_collection3_shard3_replica_n3", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard4":{ "range":"19990000-4ccbffff", "state":"active", "replicas":{"core_node9":{ "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "type":"NRT", "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/tlog", "core":"solrj_collection3_shard4_replica_n4", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard5":{ "range":"4ccc0000-7fffffff", "state":"active", "replicas":{"core_node10":{ "core":"solrj_collection3_shard5_replica_n5", "base_url":"http://127.0.0.1:41869/pzz/jk", "node_name":"127.0.0.1:41869_pzz%2Fjk", "state":"down", "type":"NRT"}}}}, "router":{ "field":"myOwnField", "name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"true", "nrtReplicas":"1", "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:40479_pzz%2Fjk, 127.0.0.1:41869_pzz%2Fjk, 127.0.0.1:34972_pzz%2Fjk, 127.0.0.1:39031_pzz%2Fjk, 127.0.0.1:39843_pzz%2Fjk] at __randomizedtesting.SeedInfo.seed([CA9ABFF875BC0E0D:42CE8022DB4063F5]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:414) at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:256) at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:144) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration Error Message: Path /autoscaling/nodeAdded/127.0.0.1:10005_solr should have been deleted Stack Trace: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10005_solr should have been deleted at __randomizedtesting.SeedInfo.seed([CA9ABFF875BC0E0D:D22037F47B89C3E2]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.junit.Assert.assertFalse(Assert.java:68) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) Build Log: [...truncated 12560 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> 639659 INFO (SUITE-TestTriggerIntegration-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_CA9ABFF875BC0E0D-001/init-core-data-001 [junit4] 2> 639660 INFO (SUITE-TestTriggerIntegration-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 639661 INFO (SUITE-TestTriggerIntegration-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) [junit4] 2> 639809 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState [junit4] 2> 639810 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 639810 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 639810 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 639814 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 639814 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 639814 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1 [junit4] 2> 639814 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1 [junit4] 2> 639844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 639844 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1 [junit4] 2> 639846 DEBUG (ScheduledTrigger-690-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639866 DEBUG (ScheduledTrigger-690-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639893 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639915 DEBUG (ScheduledTrigger-690-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639941 DEBUG (ScheduledTrigger-690-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639965 DEBUG (ScheduledTrigger-690-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 639968 DEBUG (simCloudManagerPool-689-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 639968 DEBUG (simCloudManagerPool-689-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 639982 DEBUG (simCloudManagerPool-689-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 639982 DEBUG (simCloudManagerPool-689-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 639982 DEBUG (simCloudManagerPool-689-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 639982 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 639982 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 639983 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 639983 DEBUG (ScheduledTrigger-690-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 639984 DEBUG (ScheduledTrigger-690-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 639985 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640006 DEBUG (ScheduledTrigger-690-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640006 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 640006 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640006 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 1080071227762113 [junit4] 2> 640030 DEBUG (ScheduledTrigger-690-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640030 DEBUG (ScheduledTrigger-690-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 640030 DEBUG (ScheduledTrigger-690-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640033 DEBUG (simCloudManagerPool-689-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 640036 DEBUG (simCloudManagerPool-689-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 640036 DEBUG (simCloudManagerPool-689-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 640036 DEBUG (simCloudManagerPool-689-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 640036 DEBUG (simCloudManagerPool-689-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 640036 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 640036 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 640037 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10002_solr [junit4] 2> 640037 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 640037 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 640037 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640038 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10002_solr] added at times [1080072739350613], now=1080072752686263 [junit4] 2> 640039 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65207f5e855Td672z1bsrjl65rizslw7nvy68", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":1080072739350613, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080072739350613], [junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}} [junit4] 2> 640039 DEBUG (ScheduledTrigger-690-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger] [junit4] 2> 640040 DEBUG (AutoscalingActionExecutor-691-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65207f5e855Td672z1bsrjl65rizslw7nvy68", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":1080072739350613, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080072739350613], [junit4] 2> "_enqueue_time_":1080072858411763, [junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}} [junit4] 2> 640040 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState [junit4] 2> 640040 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 640041 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 640042 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 640042 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 640042 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 640042 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 640042 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 640055 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 640055 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 640055 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 640056 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger [junit4] 2> 640056 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 640071 DEBUG (AutoscalingActionExecutor-691-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 640071 DEBUG (AutoscalingActionExecutor-691-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 100ms [junit4] 2> 640071 DEBUG (AutoscalingActionExecutor-691-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 1564 ms for event id=3d65207f5e855Td672z1bsrjl65rizslw7nvy68 [junit4] 2> 640073 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue [junit4] 2> 640073 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 640073 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 640073 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 640074 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 640075 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640075 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted [junit4] 2> java.lang.InterruptedException [junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) [junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) [junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 640078 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 640078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 640078 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 640092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr] [junit4] 2> 640092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion -1 [junit4] 2> 640092 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5 [junit4] 2> 640111 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640111 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5 [junit4] 2> 640119 DEBUG (ScheduledTrigger-693-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640139 DEBUG (ScheduledTrigger-693-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640168 DEBUG (ScheduledTrigger-693-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640188 DEBUG (ScheduledTrigger-693-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640189 DEBUG (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 640189 DEBUG (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 640191 DEBUG (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr] [junit4] 2> 640191 INFO (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 640191 DEBUG (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10002_solr] [junit4] 2> 640191 DEBUG (simCloudManagerPool-692-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 640191 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 640192 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 640192 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 640192 DEBUG (ScheduledTrigger-693-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 640192 DEBUG (ScheduledTrigger-693-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 640209 DEBUG (ScheduledTrigger-693-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640213 DEBUG (ScheduledTrigger-693-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 640213 DEBUG (ScheduledTrigger-693-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640214 DEBUG (ScheduledTrigger-693-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 1080081587985663 [junit4] 2> 640229 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640245 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 640245 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640245 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [1080081587985663], now=1080083154548363 [junit4] 2> 640245 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 640246 DEBUG (ScheduledTrigger-693-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 640267 DEBUG (AutoscalingActionExecutor-694-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "_enqueue_time_":1080083194213063, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 640267 INFO (AutoscalingActionExecutor-694-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "_enqueue_time_":1080083194213063, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 640267 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 640268 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 640268 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 640268 WARN (AutoscalingActionExecutor-694-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:525) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:309) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> 640268 DEBUG (AutoscalingActionExecutor-694-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 53 ms for event id=3d65417615cffTd672z1bsrjl65rizslw7nvy6b [junit4] 2> 640350 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 640351 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 640351 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10001_solr [junit4] 2> 640366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 640366 DEBUG (ScheduledTrigger-696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640366 DEBUG (ScheduledTrigger-696-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr [junit4] 2> 640366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 640380 DEBUG (ScheduledTrigger-696-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "_dequeue_time_":1080089222757313, [junit4] 2> "_enqueue_time_":1080083194213063, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 640380 DEBUG (ScheduledTrigger-696-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 640381 DEBUG (ScheduledTrigger-696-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 640381 DEBUG (ScheduledTrigger-696-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 640381 DEBUG (AutoscalingActionExecutor-697-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "_dequeue_time_":1080089222757313, [junit4] 2> "_enqueue_time_":1080083194213063, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 640381 INFO (AutoscalingActionExecutor-697-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"3d65417615cffTd672z1bsrjl65rizslw7nvy6b", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":1080081587985663, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080081587985663], [junit4] 2> "_dequeue_time_":1080089222757313, [junit4] 2> "_enqueue_time_":1080083194213063, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 640382 DEBUG (AutoscalingActionExecutor-697-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 640382 DEBUG (AutoscalingActionExecutor-697-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 640382 DEBUG (AutoscalingActionExecutor-697-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 81 ms for event id=3d65417615cffTd672z1bsrjl65rizslw7nvy6b [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10001_solr [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 3 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 640383 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 640403 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration [junit4] 2> 640403 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 640403 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr [junit4] 2> 640403 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 640403 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 640404 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 640420 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 640423 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 640423 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 640423 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion -1 [junit4] 2> 640423 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 640424 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640424 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 640425 DEBUG (ScheduledTrigger-699-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640455 DEBUG (ScheduledTrigger-699-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640476 DEBUG (ScheduledTrigger-699-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640496 DEBUG (ScheduledTrigger-699-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640517 DEBUG (ScheduledTrigger-699-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640522 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1 [junit4] 2> 640522 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 640523 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion -1 [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10002_solr [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 640524 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640524 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10002_solr [junit4] 2> 640546 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 640561 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640582 DEBUG (ScheduledTrigger-702-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640602 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640623 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS [junit4] 2> 640630 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640643 DEBUG (simCloudManagerPool-701-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 640644 DEBUG (simCloudManagerPool-701-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 640644 DEBUG (simCloudManagerPool-701-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640644 DEBUG (simCloudManagerPool-701-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640644 DEBUG (simCloudManagerPool-701-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 640644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9 [junit4] 2> 640644 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 640644 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9 [junit4] 2> 640644 DEBUG (ScheduledTrigger-702-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 640644 DEBUG (ScheduledTrigger-702-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640645 DEBUG (simCloudManagerPool-701-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true} [junit4] 2> 640645 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 640645 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init [junit4] 2> 640646 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 640646 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 [junit4] 2> 640646 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1 [junit4] 2> 640658 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640665 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 640665 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640665 DEBUG (ScheduledTrigger-702-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 1080104163744763 [junit4] 2> 640670 DEBUG (ScheduledTrigger-702-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 3 [junit4] 2> 640678 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 640690 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 640691 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 640691 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [1080104163744763], now=1080105439193363 [junit4] 2> 640691 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65959001bfbTd672z1bsrjl65rizslw7nvy6f", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1080104163744763, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080104163744763], [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 640711 DEBUG (ScheduledTrigger-702-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger, node_added_trigger] [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeMarkersRegistration [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 640746 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10002_solr [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 640747 INFO (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory. [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeMarkersRegistration -Dtests.seed=CA9ABFF875BC0E0D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sl-SI -Dtests.timezone=Canada/East-Saskatchewan -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1 [junit4] FAILURE 0.37s J2 | TestTriggerIntegration.testNodeMarkersRegistration <<< [junit4] > Throwable #1: java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10005_solr should have been deleted [junit4] > at __randomizedtesting.SeedInfo.seed([CA9ABFF875BC0E0D:D22037F47B89C3E2]:0) [junit4] > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeMarkersRegistration(TestTriggerIntegration.java:842) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] 2> 640788 DEBUG (AutoscalingActionExecutor-703-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65959001bfbTd672z1bsrjl65rizslw7nvy6f", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":1080104163744763, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1080104163744763], [junit4] 2> "_enqueue_time_":1080106463428363, [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 640805 DEBUG (AutoscalingActionExecutor-703-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 640805 DEBUG (AutoscalingActionExecutor-703-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms [junit4] 2> 640805 DEBUG (AutoscalingActionExecutor-703-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms [junit4] 2> 640805 DEBUG (AutoscalingActionExecutor-703-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 813 ms for event id=3d65959001bfbTd672z1bsrjl65rizslw7nvy6f [junit4] 2> 640812 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate [junit4] 2> 640812 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 640813 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10002_solr [junit4] 2> 640813 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 640813 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 640813 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 640822 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 640822 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 640842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion -1 [junit4] 2> 640842 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 640853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640853 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 640853 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640873 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640894 DEBUG (ScheduledTrigger-705-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640914 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640942 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640962 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640979 DEBUG (simCloudManagerPool-704-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 640979 DEBUG (simCloudManagerPool-704-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 640982 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 640991 DEBUG (simCloudManagerPool-704-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 640991 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 640992 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 640992 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 641011 DEBUG (ScheduledTrigger-705-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641035 DEBUG (ScheduledTrigger-705-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641036 DEBUG (simCloudManagerPool-704-thread-5) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 641036 DEBUG (simCloudManagerPool-704-thread-5) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 641036 DEBUG (simCloudManagerPool-704-thread-5) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 641036 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14 [junit4] 2> 641036 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 641036 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14 [junit4] 2> 641060 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641080 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641108 DEBUG (ScheduledTrigger-705-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641114 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65db912eaf1Td672z1bsrjl65rizslw7nvy6i", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":1080122955459313, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10004_solr":250.0, [junit4] 2> "127.0.0.1:10003_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10004_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10003_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}}} [junit4] 2> 641115 DEBUG (ScheduledTrigger-705-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, search_rate_trigger] [junit4] 2> 641134 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65db912eaf1Td672z1bsrjl65rizslw7nvy6i", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":1080122955459313, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10004_solr":250.0, [junit4] 2> "127.0.0.1:10003_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10004_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10003_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}, [junit4] 2> "_enqueue_time_":1080126642384413}} [junit4] 2> 641149 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 641149 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: search_rate_trigger after 100ms [junit4] 2> 641149 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 774 ms for event id=3d65db912eaf1Td672z1bsrjl65rizslw7nvy6i [junit4] 2> 641190 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSearchRate [junit4] 2> 641190 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 641190 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 641190 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0 [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [collection1] [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 1 [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 1 [junit4] 2> 641191 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 2 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * collection1 2 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 2 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: search_rate_trigger [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] 2> 641192 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1 [junit4] 2> 641281 DEBUG (ScheduledTrigger-705-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641301 DEBUG (ScheduledTrigger-705-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 641301 DEBUG (ScheduledTrigger-705-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"3d65e6094cecfTd672z1bsrjl65rizslw7nvy6l", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":1080125765766863, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10004_solr":250.0, [junit4] 2> "127.0.0.1:10003_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10004_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10003_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}}} [junit4] 2> 641301 DEBUG (ScheduledTrigger-705-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, search_rate_trigger] [junit4] 2> 641340 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"3d65e6094cecfTd672z1bsrjl65rizslw7nvy6l", [junit4] 2> "source":"search_rate_trigger", [junit4] 2> "eventTime":1080125765766863, [junit4] 2> "eventType":"SEARCHRATE", [junit4] 2> "properties":{ [junit4] 2> "node":{ [junit4] 2> "127.0.0.1:10004_solr":250.0, [junit4] 2> "127.0.0.1:10003_solr":250.0}, [junit4] 2> "replica":[ [junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10004_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}", [junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10003_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"], [junit4] 2> "collection":{"collection1":500.0}, [junit4] 2> "shard":{"collection1":{"shard1":500.0}}, [junit4] 2> "_enqueue_time_":1080135982598013}} [junit4] 2> 641342 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger [junit4] 2> 641342 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 641342 INFO (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 641342 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 641344 WARN (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system [junit4] 2> java.io.IOException: java.lang.InterruptedException [junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:460) [junit4] 2> at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:122) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:744) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:714) [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:312) [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [junit4] 2> at java.lang.Thread.run(Thread.java:748) [junit4] 2> Caused by: java.lang.InterruptedException [junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) [junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) [junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:458) [junit4] 2> ... 10 more [junit4] 2> 641344 DEBUG (AutoscalingActionExecutor-706-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 216 ms for event id=3d65e6094cecfTd672z1bsrjl65rizslw7nvy6l [junit4] 2> 641344 DEBUG (TEST-TestTriggerIntegration.testNodeLostTrigger-seed#[CA9ABFF875BC0E0D]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 641346 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger [junit4] 2> 641347 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 641355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10003_solr] [junit4] 2> 641355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion -1 [junit4] 2> 641355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 641355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 641355 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 641355 DEBUG (ScheduledTrigger-708-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLos [...truncated too long message...] ] > "shard1":{ [junit4] > "range":"80000000-b332ffff", [junit4] > "state":"active", [junit4] > "replicas":{"core_node6":{ [junit4] > "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/", [junit4] > "base_url":"http://127.0.0.1:41869/pzz/jk", [junit4] > "node_name":"127.0.0.1:41869_pzz%2Fjk", [junit4] > "type":"NRT", [junit4] > "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node6/data/tlog", [junit4] > "core":"solrj_collection3_shard1_replica_n1", [junit4] > "shared_storage":"true", [junit4] > "state":"active", [junit4] > "leader":"true"}}}, [junit4] > "shard2":{ [junit4] > "range":"b3330000-e665ffff", [junit4] > "state":"active", [junit4] > "replicas":{"core_node7":{ [junit4] > "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/", [junit4] > "base_url":"http://127.0.0.1:41869/pzz/jk", [junit4] > "node_name":"127.0.0.1:41869_pzz%2Fjk", [junit4] > "type":"NRT", [junit4] > "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node7/data/tlog", [junit4] > "core":"solrj_collection3_shard2_replica_n2", [junit4] > "shared_storage":"true", [junit4] > "state":"active", [junit4] > "leader":"true"}}}, [junit4] > "shard3":{ [junit4] > "range":"e6660000-1998ffff", [junit4] > "state":"active", [junit4] > "replicas":{"core_node8":{ [junit4] > "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/", [junit4] > "base_url":"http://127.0.0.1:39843/pzz/jk", [junit4] > "node_name":"127.0.0.1:39843_pzz%2Fjk", [junit4] > "type":"NRT", [junit4] > "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node8/data/tlog", [junit4] > "core":"solrj_collection3_shard3_replica_n3", [junit4] > "shared_storage":"true", [junit4] > "state":"active", [junit4] > "leader":"true"}}}, [junit4] > "shard4":{ [junit4] > "range":"19990000-4ccbffff", [junit4] > "state":"active", [junit4] > "replicas":{"core_node9":{ [junit4] > "dataDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/", [junit4] > "base_url":"http://127.0.0.1:41869/pzz/jk", [junit4] > "node_name":"127.0.0.1:41869_pzz%2Fjk", [junit4] > "type":"NRT", [junit4] > "ulogDir":"hdfs://lucene2-us-west.apache.org:37281/solr_hdfs_home/solrj_collection3/core_node9/data/tlog", [junit4] > "core":"solrj_collection3_shard4_replica_n4", [junit4] > "shared_storage":"true", [junit4] > "state":"active", [junit4] > "leader":"true"}}}, [junit4] > "shard5":{ [junit4] > "range":"4ccc0000-7fffffff", [junit4] > "state":"active", [junit4] > "replicas":{"core_node10":{ [junit4] > "core":"solrj_collection3_shard5_replica_n5", [junit4] > "base_url":"http://127.0.0.1:41869/pzz/jk", [junit4] > "node_name":"127.0.0.1:41869_pzz%2Fjk", [junit4] > "state":"down", [junit4] > "type":"NRT"}}}}, [junit4] > "router":{ [junit4] > "field":"myOwnField", [junit4] > "name":"compositeId"}, [junit4] > "maxShardsPerNode":"1", [junit4] > "autoAddReplicas":"true", [junit4] > "nrtReplicas":"1", [junit4] > "tlogReplicas":"0"} with /live_nodes: [127.0.0.1:40479_pzz%2Fjk, 127.0.0.1:41869_pzz%2Fjk, 127.0.0.1:34972_pzz%2Fjk, 127.0.0.1:39031_pzz%2Fjk, 127.0.0.1:39843_pzz%2Fjk] [junit4] > at __randomizedtesting.SeedInfo.seed([CA9ABFF875BC0E0D:42CE8022DB4063F5]:0) [junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:414) [junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:256) [junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:144) [junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993) [junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] 2> 3728861 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called [junit4] 2> 3728889 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 [junit4] 2> 3728990 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37281) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted [junit4] 2> 3728991 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37281) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1657909561-127.0.0.1-1516625729876 (Datanode Uuid e3707300-e809-40da-bd43-0b0769dc4153) service to lucene2-us-west.apache.org/127.0.0.1:37281 [junit4] 2> 3728992 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called [junit4] 2> 3729002 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 [junit4] 2> 3729103 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37281) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted [junit4] 2> 3729104 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37281) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1657909561-127.0.0.1-1516625729876 (Datanode Uuid 7946e6f8-dd72-41f6-a721-b06502f2b94b) service to lucene2-us-west.apache.org/127.0.0.1:37281 [junit4] 2> 3729112 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[CA9ABFF875BC0E0D]-worker) [ ] o.m.log Stopped HttpServer2$selectchannelconnectorwithsafestar...@lucene2-us-west.apache.org:0 [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_CA9ABFF875BC0E0D-001 [junit4] 2> Jan 22, 2018 1:46:52 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4936351c), locale=es-UY, timezone=Asia/Hovd [junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=4,free=169358856,total=524812288 [junit4] 2> NOTE: All tests run in this JVM: [TestJsonFacetRefinement, HdfsRestartWhileUpdatingTest, SpatialHeatmapFacetsTest, DateMathParserTest, TestFreeTextSuggestions, SearchHandlerTest, NodeLostTriggerTest, TestTriggerIntegration, SharedFSAutoReplicaFailoverTest] [junit4] Completed [226/768 (2!)] on J2 in 3092.57s, 1 test, 1 failure <<< FAILURES! [...truncated 49942 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org