Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/198/
3 tests failed. FAILED: org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test Error Message: Error from server at https://127.0.0.1:40759/solr: Could not restore core Stack Trace: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40759/solr: Could not restore core at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:1BC661B962ADC94C]:0) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483) at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413) at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211) at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:287) at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:142) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger Error Message: The trigger did not fire at all Stack Trace: java.lang.AssertionError: The trigger did not fire at all at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:B5F8E05AD75E0BD8]:0) at org.junit.Assert.fail(Assert.java:93) at org.junit.Assert.assertTrue(Assert.java:43) at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger(TestTriggerIntegration.java:408) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.lang.Thread.run(Thread.java:748) FAILED: org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit Error Message: Java heap space Stack Trace: java.lang.OutOfMemoryError: Java heap space at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:A02076A7C1E67E03]:0) at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:424) at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:129) at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:113) at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:292) at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:372) at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113) at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:78) at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:197) at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4628) at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4144) at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40) at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2335) at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3570) at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3522) at org.apache.lucene.index.RandomIndexWriter.maybeFlushOrCommit(RandomIndexWriter.java:216) at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:188) at org.apache.solr.uninverting.TestDocTermOrds.testTriggerUnInvertLimit(TestDocTermOrds.java:171) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) Build Log: [...truncated 12986 lines...] [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration [junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_93925E63CC51A4B4-001/init-core-data-001 [junit4] 2> 172481 WARN (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1 [junit4] 2> 172482 INFO (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false [junit4] 2> 172516 INFO (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN) [junit4] 2> 172533 INFO (SUITE-TestTriggerIntegration-seed#[93925E63CC51A4B4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 172791 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers [junit4] 2> 172861 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0 [junit4] 2> 172863 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1 [junit4] 2> 172863 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0 [junit4] IGNOR/A 0.08s J2 | TestTriggerIntegration.testSearchRate [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] 2> 172970 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 173009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 173009 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0 [junit4] 2> 173043 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger [junit4] 2> 173043 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1 [junit4] 2> 173066 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 173067 DEBUG (ScheduledTrigger-107-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173079 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 173174 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now [junit4] 2> 173174 DEBUG (ScheduledTrigger-107-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now [junit4] 2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor [junit4] 2> 173175 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor [junit4] 2> 173189 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely [junit4] 2> 173189 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 173202 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers [junit4] 2> 173202 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2 [junit4] 2> 173203 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1 [junit4] 2> 173203 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2 [junit4] 2> 173222 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 173223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 173223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2 [junit4] 2> 173223 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173294 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173332 DEBUG (ScheduledTrigger-110-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173384 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173409 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173444 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173482 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173517 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173554 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173590 DEBUG (simCloudManagerPool-109-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 173590 DEBUG (simCloudManagerPool-109-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3 [junit4] 2> 173590 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3 [junit4] 2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 173605 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3 [junit4] 2> 173643 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173663 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173700 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 173725 DEBUG (simCloudManagerPool-109-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 173725 DEBUG (simCloudManagerPool-109-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4 [junit4] 2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4 [junit4] 2> 173733 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr] [junit4] 2> 173733 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 173734 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4 [junit4] 2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 173737 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 5807910489118412 [junit4] 2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 173773 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 173774 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10002_solr] added at times [5807910489118412], now=5807912291932112 [junit4] 2> 173776 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":5807910489118412, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5807910489118412], [junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}} [junit4] 2> 174066 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger] [junit4] 2> 174191 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 174192 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 174193 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 174193 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 174193 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 174193 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 174194 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 174194 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 174194 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 174194 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 174214 DEBUG (ScheduledTrigger-110-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10002_solr [junit4] 2> 174227 DEBUG (AutoscalingActionExecutor-111-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw", [junit4] 2> "source":"node_added_trigger", [junit4] 2> "eventTime":5807910489118412, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5807910489118412], [junit4] 2> "_enqueue_time_":5807926959279312, [junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}} [junit4] 2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms [junit4] 2> 174250 DEBUG (AutoscalingActionExecutor-111-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 1777 ms for event id=14a243899756ccTb6dcrwj17bt0ghdhsu6uyamuw [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory. [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeAddedTrigger -Dtests.seed=93925E63CC51A4B4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca-ES -Dtests.timezone=Pacific/Samoa -Dtests.asserts=true -Dtests.file.encoding=UTF-8 [junit4] FAILURE 1.33s J2 | TestTriggerIntegration.testNodeAddedTrigger <<< [junit4] > Throwable #1: java.lang.AssertionError: The trigger did not fire at all [junit4] > at __randomizedtesting.SeedInfo.seed([93925E63CC51A4B4:B5F8E05AD75E0BD8]:0) [junit4] > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTrigger(TestTriggerIntegration.java:408) [junit4] > at java.lang.Thread.run(Thread.java:748) [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testCooldown [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeLostTrigger [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] 2> 174394 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174520 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174540 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174576 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174812 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174813 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 174813 DEBUG (ScheduledTrigger-110-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 174885 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 174906 DEBUG (ScheduledTrigger-110-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174907 DEBUG (ScheduledTrigger-110-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 174907 DEBUG (ScheduledTrigger-110-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 174994 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 174994 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 174995 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 175141 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 175562 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 175563 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 175563 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 175598 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 175599 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger [junit4] 2> 175599 DEBUG (ScheduledTrigger-110-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 175600 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState [junit4] 2> 175600 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5 [junit4] 2> 175601 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now [junit4] 2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now [junit4] 2> 175601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor [junit4] 2> 175601 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor [junit4] 2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely [junit4] 2> 175602 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 175673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers [junit4] 2> 175673 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6 [junit4] 2> 175711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1 [junit4] 2> 175711 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6 [junit4] 2> 175727 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 175753 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 175753 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6 [junit4] 2> 175893 DEBUG (ScheduledTrigger-113-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 175907 DEBUG (simCloudManagerPool-112-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 175908 DEBUG (simCloudManagerPool-112-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7 [junit4] 2> 176192 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 176193 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7 [junit4] 2> 176227 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 176228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 176228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7 [junit4] 2> 176228 DEBUG (simCloudManagerPool-112-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 176244 DEBUG (simCloudManagerPool-112-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8 [junit4] 2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8 [junit4] 2> 176245 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 176245 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8 [junit4] 2> 176245 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 176245 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 176264 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 176266 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 176266 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 176300 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 176337 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 176338 DEBUG (simCloudManagerPool-112-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 176338 DEBUG (simCloudManagerPool-112-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9 [junit4] 2> 176339 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9 [junit4] 2> 176374 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr] [junit4] 2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true} [junit4] 2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10003_solr [junit4] 2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr [junit4] 2> 176374 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9 [junit4] 2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger [junit4] 2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 176374 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [5808042313008762], now=5808042332719662 [junit4] 2> 176375 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":5808042313008762, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808042313008762], [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 176376 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_restore_trigger] [junit4] 2> 176446 DEBUG (ScheduledTrigger-113-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr [junit4] 2> 176447 DEBUG (AutoscalingActionExecutor-114-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz", [junit4] 2> "source":"node_added_restore_trigger", [junit4] 2> "eventTime":5808042313008762, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808042313008762], [junit4] 2> "_enqueue_time_":5808042440598362, [junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 176447 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 3 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1 [junit4] 2> 176448 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 176449 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_restore_trigger [junit4] 2> 176449 INFO (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1 [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testListeners [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testEventFromRestoredState [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] 2> 176539 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue [junit4] 2> 176539 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10 [junit4] 2> 176556 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10 [junit4] 2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_restore_trigger after 100ms [junit4] 2> 176556 DEBUG (AutoscalingActionExecutor-114-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 5470 ms for event id=14a2623ae8467aTb6dcrwj17bt0ghdhsu6uyamuz [junit4] 2> 176609 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers [junit4] 2> 176610 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 176610 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10 [junit4] 2> 176610 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now [junit4] 2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now [junit4] 2> 176610 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor [junit4] 2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor [junit4] 2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely [junit4] 2> 176611 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 176612 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 176667 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers [junit4] 2> 176667 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11 [junit4] 2> 176668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion -1 [junit4] 2> 176668 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11 [junit4] 2> 176718 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 176755 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 176755 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11 [junit4] 2> 176755 DEBUG (ScheduledTrigger-116-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 176882 DEBUG (simCloudManagerPool-115-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 176883 DEBUG (simCloudManagerPool-115-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12 [junit4] 2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12 [junit4] 2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 176942 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12 [junit4] 2> 176964 INFO (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 176964 DEBUG (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 176965 DEBUG (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 176965 DEBUG (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 177066 INFO (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 177066 DEBUG (simCloudManagerPool-115-thread-2) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 177066 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr] [junit4] 2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 177067 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 177080 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 177080 DEBUG (ScheduledTrigger-116-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 177081 DEBUG (ScheduledTrigger-116-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 177081 DEBUG (ScheduledTrigger-116-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 5808077654319262 [junit4] 2> 177101 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 177153 DEBUG (ScheduledTrigger-116-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [5808077654319262], now=5808081303618062 [junit4] 2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 177154 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 177210 DEBUG (ScheduledTrigger-116-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10004_solr [junit4] 2> 177211 DEBUG (AutoscalingActionExecutor-117-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "_enqueue_time_":5808081326400762, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 177211 INFO (AutoscalingActionExecutor-117-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "_enqueue_time_":5808081326400762, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}} [junit4] 2> 177226 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now [junit4] 2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now [junit4] 2> 177226 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor [junit4] 2> 177226 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 177227 WARN (AutoscalingActionExecutor-117-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions [junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed. [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:546) ~[java/:?] [junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:319) ~[java/:?] [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_152] [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_152] [junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:192) ~[java/:?] [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_152] [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_152] [junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_152] [junit4] 2> 177227 DEBUG (AutoscalingActionExecutor-117-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 798 ms for event id=14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2 [junit4] 2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor [junit4] 2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely [junit4] 2> 177227 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 177229 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13 [junit4] 2> 177283 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated [junit4] 2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1 [junit4] 2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13 [junit4] 2> 177283 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 177284 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10000_solr [junit4] 2> 177335 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 177335 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true} [junit4] 2> 177355 DEBUG (ScheduledTrigger-119-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 177356 DEBUG (ScheduledTrigger-119-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10000_solr [junit4] 2> 177371 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13 [junit4] 2> 177372 DEBUG (ScheduledTrigger-119-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "_dequeue_time_":5808092202008612, [junit4] 2> "_enqueue_time_":5808081326400762, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 177373 DEBUG (ScheduledTrigger-119-thread-2) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger1] [junit4] 2> 177445 DEBUG (ScheduledTrigger-119-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 177445 DEBUG (ScheduledTrigger-119-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 177495 DEBUG (AutoscalingActionExecutor-120-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "_dequeue_time_":5808092202008612, [junit4] 2> "_enqueue_time_":5808081326400762, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 177496 INFO (AutoscalingActionExecutor-120-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: { [junit4] 2> "id":"14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808077654319262, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808077654319262], [junit4] 2> "_dequeue_time_":5808092202008612, [junit4] 2> "_enqueue_time_":5808081326400762, [junit4] 2> "nodeNames":["127.0.0.1:10004_solr"], [junit4] 2> "replaying":true}} [junit4] 2> 177499 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase [junit4] 2> [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############ [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############################################# [junit4] 2> [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10000_solr [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [] [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ########## [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ########### [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1 [junit4] 2> 177500 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2 [junit4] 2> 177526 DEBUG (AutoscalingActionExecutor-120-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 177542 DEBUG (AutoscalingActionExecutor-120-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 177542 DEBUG (AutoscalingActionExecutor-120-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 2335 ms for event id=14a26a7569e09eTb6dcrwj17bt0ghdhsu6uyamv2 [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeMarkersRegistration [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] IGNOR/A 0.00s J2 | TestTriggerIntegration.testNodeLostTriggerRestoreState [junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028)) [junit4] 2> 177656 DEBUG (ScheduledTrigger-119-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 177884 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling [junit4] 2> 177884 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14 [junit4] 2> 177884 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10000_solr [junit4] 2> 177884 INFO (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache... [junit4] 2> 177884 DEBUG (ScheduledTrigger-119-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 177884 DEBUG (ScheduledTrigger-119-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 177885 DEBUG (ScheduledTrigger-119-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 177885 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now [junit4] 2> 177885 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now [junit4] 2> 177886 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor [junit4] 2> 177891 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 177931 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor [junit4] 2> 177932 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely [junit4] 2> 177932 DEBUG (TEST-TestTriggerIntegration.testTriggerThrottling-seed#[93925E63CC51A4B4]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly [junit4] 2> 177987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers [junit4] 2> 177987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15 [junit4] 2> 178080 DEBUG (simCloudManagerPool-121-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 178094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion -1 [junit4] 2> 178094 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15 [junit4] 2> 178094 DEBUG (simCloudManagerPool-121-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16 [junit4] 2> 178117 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 15 [junit4] 2> 178130 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16 [junit4] 2> 178132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers [junit4] 2> 178132 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16 [junit4] 2> 178132 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178133 DEBUG (simCloudManagerPool-121-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 178133 DEBUG (simCloudManagerPool-121-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17 [junit4] 2> 178135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17 [junit4] 2> 178135 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 178135 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 178136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 178136 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178136 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178136 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 17, lastZnodeVersion 17 [junit4] 2> 178152 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178152 DEBUG (simCloudManagerPool-121-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 178153 DEBUG (simCloudManagerPool-121-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 18 [junit4] 2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178188 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178209 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178209 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178210 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178210 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 18 [junit4] 2> 178211 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10002_solr] [junit4] 2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$ThrottlingTesterAction}], enabled=true} [junit4] 2> 178211 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 18, lastZnodeVersion 18 [junit4] 2> 178211 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 178211 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 5808135434972662 [junit4] 2> 178236 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [5808135434972662], now=5808135437518362 [junit4] 2> 178237 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808135434972662, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808135434972662], [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 178237 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1] [junit4] 2> 178238 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr [junit4] 2> 178238 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6", [junit4] 2> "source":"node_added_trigger1", [junit4] 2> "eventTime":5808135434972662, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808135434972662], [junit4] 2> "_enqueue_time_":5808135484706712, [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 178238 INFO (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger1 [junit4] 2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 178254 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 783 ms for event id=14a277e968adf6Tb6dcrwj17bt0ghdhsu6uyamv6 [junit4] 2> 178362 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 178362 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178362 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 [junit4] 2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178384 DEBUG (ScheduledTrigger-122-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 178398 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 178398 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3 [junit4] 2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 5808143544140212 [junit4] 2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [5808143544140212], now=5808143549185512 [junit4] 2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: { [junit4] 2> "id":"14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9", [junit4] 2> "source":"node_added_trigger2", [junit4] 2> "eventTime":5808143544140212, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808143544140212], [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 178399 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_added_trigger2, node_added_trigger1] [junit4] 2> 178400 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10005_solr [junit4] 2> 178401 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for { [junit4] 2> "id":"14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9", [junit4] 2> "source":"node_added_trigger2", [junit4] 2> "eventTime":5808143544140212, [junit4] 2> "eventType":"NODEADDED", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[5808143544140212], [junit4] 2> "_enqueue_time_":5808143567743712, [junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}} [junit4] 2> 178401 INFO (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration last action at 5808135539962012 time = 5808143655653162 [junit4] 2> 178401 INFO (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration action executed from node_added_trigger2 [junit4] 2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms [junit4] 2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger2 after 100ms [junit4] 2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger1 after 100ms [junit4] 2> 178419 DEBUG (AutoscalingActionExecutor-123-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 913 ms for event id=14a279ccc0c1b4Tb6dcrwj17bt0ghdhsu6uyamv9 [junit4] 2> 178420 DEBUG (simCloudManagerPool-121-thread-8) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 178420 DEBUG (simCloudManagerPool-121-thread-8) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 19 [junit4] 2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 19 [junit4] 2> 178421 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger1 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr] [junit4] 2> 178421 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 19, lastZnodeVersion 19 [junit4] 2> 178422 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3 [junit4] 2> 178455 DEBUG (ScheduledTrigger-122-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3 [junit4] 2> 178456 DEBUG (simCloudManagerPool-121-thread-9) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration [junit4] 2> 178456 DEBUG (simCloudManagerPool-121-thread-9) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 20 [junit4] 2> 178457 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 20 [junit4] 2> 178457 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init [junit4] 2> 178457 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger2 - Initial livenodes: [127.0.0.1:10004_solr, 127.0.0.1:10005_solr, 127.0.0.1:10002_solr] [junit4] 2> 178458 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 20, lastZnodeVersion 20 [junit4] 2> 178458 DEBUG (ScheduledTrigger-122-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3 [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] 2> 178528 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178564 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 [junit4] 2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2 [junit4] 2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2 [junit4] 2> 178565 DEBUG (ScheduledTrigger-122-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1 [junit4] [...truncated too long message...] 987-thread-3) [n:127.0.0.1:42668_solr c:hdfsbackuprestore s:shard2 r:core_node12 x:hdfsbackuprestore_shard2_replica_p10] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard2.leader, tag=3f9292a8 [junit4] 2> 6718968 INFO (coreCloseExecutor-13986-thread-1) [n:127.0.0.1:40759_solr c:hdfsbackuprestore s:shard2 r:core_node11 x:hdfsbackuprestore_shard2_replica_t8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard2.leader, tag=a7c2bfd [junit4] 2> 6718999 INFO (jetty-closer-57564-thread-2) [ ] o.a.s.c.Overseer Overseer (id=72438645494841350-127.0.0.1:40759_solr-n_0000000000) closing [junit4] 2> 6718999 INFO (OverseerStateUpdate-72438645494841350-127.0.0.1:40759_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40759_solr [junit4] 2> 6719000 WARN (OverseerAutoScalingTriggerThread-72438645494841350-127.0.0.1:40759_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 6719003 INFO (jetty-closer-57564-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@381f9003{/solr,null,UNAVAILABLE} [junit4] 2> 6719003 INFO (jetty-closer-57564-thread-1) [ ] o.e.j.s.session Stopped scavenging [junit4] 2> 6719004 INFO (jetty-closer-57564-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6298e1cc{/solr,null,UNAVAILABLE} [junit4] 2> 6719005 INFO (jetty-closer-57564-thread-2) [ ] o.e.j.s.session Stopped scavenging [junit4] 2> 6719006 ERROR (SUITE-TestHdfsCloudBackupRestore-seed#[93925E63CC51A4B4]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 6719014 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[93925E63CC51A4B4]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45419 45419 [junit4] 2> 6719074 INFO (Thread-10622) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45419 45419 [junit4] 2> 6719075 WARN (Thread-10622) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 7 /solr/collections/hdfsbackuprestore/terms/shard2 [junit4] 2> 5 /solr/collections/hdfsbackuprestore/terms/shard1 [junit4] 2> 5 /solr/aliases.json [junit4] 2> 3 /solr/collections/hdfsbackuprestore_restored/terms/shard1 [junit4] 2> 2 /solr/security.json [junit4] 2> 2 /solr/configs/conf1 [junit4] 2> 2 /solr/collections/hdfsbackuprestore_restored/terms/shard2 [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 16 /solr/collections/hdfsbackuprestore/state.json [junit4] 2> 15 /solr/collections/hdfsbackuprestore_restored/state.json [junit4] 2> 5 /solr/clusterstate.json [junit4] 2> 5 /solr/clusterprops.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 5 /solr/live_nodes [junit4] 2> 5 /solr/collections [junit4] 2> [junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_93925E63CC51A4B4-001 [junit4] 2> Apr 11, 2018 8:03:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2> WARNING: Will linger awaiting termination of 39 leaked thread(s). [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {shard_s=PostingsFormat(name=LuceneFixedGap), id=FST50}, docValues:{}, maxPointsInLeafNode=1843, maxMBSortInHeap=7.6243392989430845, sim=RandomSimilarity(queryNorm=false): {}, locale=uk-UA, timezone=Africa/Asmera [junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=149539104,total=479723520 [junit4] 2> NOTE: All tests run in this JVM: [TestSmileRequest, SchemaVersionSpecificBehaviorTest, TestReload, TestNumericTokenStream, ZkShardTermsTest, RegexBytesRefFilterTest, TestDistributedGrouping, ResourceLoaderTest, TestRTGBase, TestScoreJoinQPScore, TestAuthenticationFramework, TestSweetSpotSimilarityFactory, ReplicaListTransformerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestFuzzyAnalyzedSuggestions, FastVectorHighlighterTest, TestOnReconnectListenerSupport, XsltUpdateRequestHandlerTest, SyncSliceTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestReversedWildcardFilterFactory, TestUseDocValuesAsStored, TestSkipOverseerOperations, TestFoldingMultitermQuery, TestLuceneIndexBackCompat, DistributedQueueTest, TestMaxScoreQueryParser, TestCustomStream, TestBackupRepositoryFactory, TestJsonFacetsWithNestedObjects, SuggestComponentContextFilterQueryTest, TestPartialUpdateDeduplication, PeerSyncTest, TestIntervalFaceting, TestZkAclsWithHadoopAuth, TestCursorMarkWithoutUniqueKey, CurrencyFieldTypeTest, BasicAuthStandaloneTest, SuggesterTest, OverseerModifyCollectionTest, TestNumericTerms64, TestCloudPseudoReturnFields, WrapperMergePolicyFactoryTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestReloadDeadlock, TestDelegationWithHadoopAuth, TestSchemaNameResource, TestSubQueryTransformerDistrib, HdfsUnloadDistributedZkTest, JsonLoaderTest, SignatureUpdateProcessorFactoryTest, TestSimpleTextCodec, TestRTimerTree, TestExportWriter, TestFieldCacheSort, TestSchemaManager, CollectionTooManyReplicasTest, TestComplexPhraseLeadingWildcard, SOLR749Test, ZkControllerTest, LeaderFailoverAfterPartitionTest, TestSolrJ, SortSpecParsingTest, TestDistribDocBasedVersion, NodeAddedTriggerTest, BufferStoreTest, TestPayloadScoreQParserPlugin, LargeFieldTest, TestReplicationHandler, TestDefaultStatsCache, TestPKIAuthenticationPlugin, TestCloudNestedDocsSort, DirectSolrConnectionTest, CloudExitableDirectoryReaderTest, MetricsConfigTest, AnalyticsQueryTest, TestBM25SimilarityFactory, TestSearchPerf, TestExpandComponent, EnumFieldTest, TriggerCooldownIntegrationTest, TestSimDistributedQueue, TestMultiValuedNumericRangeQuery, TestNRTOpen, SecurityConfHandlerTest, SuggesterFSTTest, HdfsChaosMonkeyNothingIsSafeTest, TestHighFrequencyDictionaryFactory, ExitableDirectoryReaderTest, MetricTriggerIntegrationTest, ParsingFieldUpdateProcessorsTest, TestBlendedInfixSuggestions, TestRecovery, LegacyCloudClusterPropTest, TestLeaderInitiatedRecoveryThread, TestCoreAdminApis, IndexSchemaTest, HighlighterTest, SchemaApiFailureTest, TestRestManager, TestQueryUtils, TestFieldCacheWithThreads, TestConfigSetsAPIZkFailure, SolrCLIZkUtilsTest, TestSortByMinMaxFunction, TestTolerantUpdateProcessorRandomCloud, TestInPlaceUpdatesStandalone, DocumentBuilderTest, DistributedFacetPivotWhiteBoxTest, HdfsRecoverLeaseTest, DistributedFacetSimpleRefinementLongTailTest, TestSolrConfigHandlerConcurrent, SearchRateTriggerTest, TestClusterProperties, MultiSolrCloudTestCaseTest, TestRawTransformer, ShardSplitTest, ActionThrottleTest, TermsComponentTest, TestDocTermOrds, ConnectionManagerTest, DistributedIntervalFacetingTest, TestSolrFieldCacheBean, CoreAdminOperationTest, CloneFieldUpdateProcessorFactoryTest, V2StandaloneTest, TestExecutePlanAction, QueryElevationComponentTest, SortByFunctionTest, TestSolrCloudSnapshots, SliceStateTest, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, TestSort, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, TestBadConfig, SoftAutoCommitTest, CSVRequestHandlerTest, TestLFUCache, TestCrossCoreJoin, TestDistributedMissingSort, TokenizerChainTest, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, AssignBackwardCompatibilityTest, CleanupOldIndexTest, ConfigSetsAPITest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DocValuesNotIndexedTest, ForceLeaderTest, LIRRollingUpdatesTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestPullReplicaErrorHandling, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestShortCircuitedRequests, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, TriLevelCompositeIdRoutingTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, CustomCollectionTest, SimpleCollectionCreateDeleteTest, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore] [junit4] Completed [604/800 (3!)] on J1 in 25.97s, 1 test, 1 error <<< FAILURES! [...truncated 657 lines...] [junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J1-20180411_181116_5877357961297125018620.sysout [junit4] >>> JVM J1 emitted unexpected output (verbatim) ---- [junit4] java.lang.OutOfMemoryError: Java heap space [junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/heapdumps/java_pid6943.hprof ... [junit4] Heap dump file created [475496812 bytes in 3.465 secs] [junit4] <<< JVM J1: EOF ---- [...truncated 9284 lines...] BUILD FAILED /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:651: The following error occurred while executing this line: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created: * java_pid6943.hprof Total time: 495 minutes 19 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org
