Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4291/ Java: 64bit/jdk-9 -XX:+UseCompressedOops -XX:+UseParallelGC
4 tests failed. FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[E66D3DABF20F30F0:FD7DCCE704FFC8C0]} Error Message: Could not find collection : pull_replica_test_create_delete Stack Trace: org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete at __randomizedtesting.SeedInfo.seed([E66D3DABF20F30F0:FD7DCCE704FFC8C0]:0) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:111) at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:161) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[E66D3DABF20F30F0:492B701B303D03EC]} Error Message: Could not find collection : pull_replica_test_create_delete Stack Trace: org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete at __randomizedtesting.SeedInfo.seed([E66D3DABF20F30F0:492B701B303D03EC]:0) at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:111) at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247) at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:161) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: org.apache.solr.cloud.TestPullReplica.testAddDocs Error Message: Expected collection to be created with 1 shard and 4 replicas null Live Nodes: [127.0.0.1:49207_solr, 127.0.0.1:49208_solr] Last available state: null Stack Trace: java.lang.AssertionError: Expected collection to be created with 1 shard and 4 replicas null Live Nodes: [127.0.0.1:49207_solr, 127.0.0.1:49208_solr] Last available state: null at __randomizedtesting.SeedInfo.seed([E66D3DABF20F30F0:6E626AD1141C21D]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269) at org.apache.solr.cloud.TestPullReplica.testAddDocs(TestPullReplica.java:220) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication Error Message: Index 0 out-of-bounds for length 0 Stack Trace: java.lang.IndexOutOfBoundsException: Index 0 out-of-bounds for length 0 at __randomizedtesting.SeedInfo.seed([E66D3DABF20F30F0:F22566FED1088DEE]:0) at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) at java.base/java.util.Objects.checkIndex(Objects.java:372) at java.base/java.util.ArrayList.get(ArrayList.java:439) at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:564) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:844) Build Log: [...truncated 12714 lines...] [junit4] Suite: org.apache.solr.cloud.TestPullReplica [junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/init-core-data-001 [junit4] 2> 1408538 WARN (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=37 numCloses=37 [junit4] 2> 1408538 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true [junit4] 2> 1408542 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0) w/ MAC_OS_X supressed clientAuth [junit4] 2> 1408542 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom [junit4] 2> 1408543 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001 [junit4] 2> 1408543 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 1408543 INFO (Thread-2779) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 1408544 INFO (Thread-2779) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 1408547 ERROR (Thread-2779) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes [junit4] 2> 1408648 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:49205 [junit4] 2> 1408675 INFO (jetty-launcher-3279-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 1408675 INFO (jetty-launcher-3279-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531 [junit4] 2> 1408679 INFO (jetty-launcher-3279-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c96b7b3{/solr,null,AVAILABLE} [junit4] 2> 1408679 INFO (jetty-launcher-3279-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e1c0760{/solr,null,AVAILABLE} [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7705d{HTTP/1.1,[http/1.1]}{127.0.0.1:49208} [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5ba7b9fc{HTTP/1.1,[http/1.1]}{127.0.0.1:49207} [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-1) [ ] o.e.j.s.Server Started @1410448ms [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-2) [ ] o.e.j.s.Server Started @1410448ms [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49207} [junit4] 2> 1408684 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49208} [junit4] 2> 1408685 ERROR (jetty-launcher-3279-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 1408685 ERROR (jetty-launcher-3279-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete. [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0 [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0 [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-16T06:27:38.256580Z [junit4] 2> 1408685 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-16T06:27:38.256615Z [junit4] 2> 1408688 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 1408688 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading... [junit4] 2> 1408700 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49205/solr [junit4] 2> 1408700 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49205/solr [junit4] 2> 1408836 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 1408836 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 1408837 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49208_solr [junit4] 2> 1408837 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49207_solr [junit4] 2> 1408838 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99012683898355717-127.0.0.1:49208_solr-n_0000000000) starting [junit4] 2> 1408839 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 1408839 INFO (zkCallback-3290-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 1408851 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49208_solr [junit4] 2> 1408852 INFO (zkCallback-3290-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 1408853 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2) [junit4] 2> 1408874 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49207.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408881 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49207.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408881 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49207.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408882 INFO (jetty-launcher-3279-thread-1) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node1/. [junit4] 2> 1408882 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49208.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408889 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49208.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408889 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49208.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1408890 INFO (jetty-launcher-3279-thread-2) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node2/. [junit4] 2> 1408916 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2) [junit4] 2> 1408917 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:49205/solr ready [junit4] 2> 1408922 INFO (SUITE-TestPullReplica-seed#[E66D3DABF20F30F0]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false [junit4] 2> 1408925 INFO (qtp1318894667-17244) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 1408927 INFO (qtp1318894667-17244) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1 [junit4] 2> 1408937 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testKillLeader [junit4] 2> 1408938 INFO (qtp1318894667-17245) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_kill_leader&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true [junit4] 2> 1408941 INFO (OverseerThreadFactory-7487-thread-1) [ ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_kill_leader [junit4] 2> 1409058 INFO (OverseerStateUpdate-99012683898355717-127.0.0.1:49208_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_kill_leader", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "type":"NRT", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 1409066 INFO (OverseerStateUpdate-99012683898355717-127.0.0.1:49208_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() { [junit4] 2> "operation":"ADDREPLICA", [junit4] 2> "collection":"pull_replica_test_kill_leader", [junit4] 2> "shard":"shard1", [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "type":"PULL", [junit4] 2> "waitForFinalState":"false"} [junit4] 2> 1409275 INFO (qtp1318894667-17250) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_leader_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_kill_leader&shard=shard1&wt=javabin&version=2&replicaType=NRT [junit4] 2> 1409276 INFO (qtp1318894667-17250) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 1409276 INFO (qtp1879748196-17240) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_leader_shard1_replica_p2&action=CREATE&numShards=1&collection=pull_replica_test_kill_leader&shard=shard1&wt=javabin&version=2&replicaType=PULL [junit4] 2> 1409276 INFO (qtp1879748196-17240) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores [junit4] 2> 1409392 INFO (zkCallback-3290-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1409392 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1409392 INFO (zkCallback-3290-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1409393 INFO (zkCallback-3291-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410302 INFO (qtp1879748196-17240) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0 [junit4] 2> 1410302 INFO (qtp1318894667-17250) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0 [junit4] 2> 1410323 INFO (qtp1879748196-17240) [ ] o.a.s.s.IndexSchema [pull_replica_test_kill_leader_shard1_replica_p2] Schema name=minimal [junit4] 2> 1410324 INFO (qtp1318894667-17250) [ ] o.a.s.s.IndexSchema [pull_replica_test_kill_leader_shard1_replica_n1] Schema name=minimal [junit4] 2> 1410325 INFO (qtp1879748196-17240) [ ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 1410325 INFO (qtp1879748196-17240) [ ] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_kill_leader_shard1_replica_p2' using configuration from collection pull_replica_test_kill_leader, trusted=true [junit4] 2> 1410325 INFO (qtp1879748196-17240) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49207.solr.core.pull_replica_test_kill_leader.shard1.replica_p2' (registry 'solr.core.pull_replica_test_kill_leader.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1410326 INFO (qtp1879748196-17240) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 1410326 INFO (qtp1879748196-17240) [ ] o.a.s.c.SolrCore [[pull_replica_test_kill_leader_shard1_replica_p2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node1/pull_replica_test_kill_leader_shard1_replica_p2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node1/./pull_replica_test_kill_leader_shard1_replica_p2/data/] [junit4] 2> 1410326 INFO (qtp1318894667-17250) [ ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id [junit4] 2> 1410326 INFO (qtp1318894667-17250) [ ] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_kill_leader_shard1_replica_n1' using configuration from collection pull_replica_test_kill_leader, trusted=true [junit4] 2> 1410327 INFO (qtp1318894667-17250) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49208.solr.core.pull_replica_test_kill_leader.shard1.replica_n1' (registry 'solr.core.pull_replica_test_kill_leader.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2663b676 [junit4] 2> 1410327 INFO (qtp1318894667-17250) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder [junit4] 2> 1410327 INFO (qtp1318894667-17250) [ ] o.a.s.c.SolrCore [[pull_replica_test_kill_leader_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node2/pull_replica_test_kill_leader_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_E66D3DABF20F30F0-001/tempDir-001/node2/./pull_replica_test_kill_leader_shard1_replica_n1/data/] [junit4] 2> 1410366 INFO (qtp1879748196-17240) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 1410366 INFO (qtp1879748196-17240) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 1410367 INFO (qtp1318894667-17250) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog [junit4] 2> 1410367 INFO (qtp1318894667-17250) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536 [junit4] 2> 1410367 INFO (qtp1879748196-17240) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1480409[pull_replica_test_kill_leader_shard1_replica_p2] main] [junit4] 2> 1410368 INFO (qtp1318894667-17250) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled [junit4] 2> 1410368 INFO (qtp1318894667-17250) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled [junit4] 2> 1410368 INFO (qtp1879748196-17240) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 1410369 INFO (qtp1879748196-17240) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 1410369 INFO (qtp1318894667-17250) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c5fb61a[pull_replica_test_kill_leader_shard1_replica_n1] main] [junit4] 2> 1410369 INFO (qtp1879748196-17240) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms. [junit4] 2> 1410370 INFO (searcherExecutor-7492-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_p2] Registered new searcher Searcher@1480409[pull_replica_test_kill_leader_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 1410370 INFO (qtp1879748196-17240) [ ] o.a.s.c.ZkController pull_replica_test_kill_leader_shard1_replica_p2 starting background replication from leader [junit4] 2> 1410370 INFO (qtp1879748196-17240) [ ] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03 [junit4] 2> 1410370 INFO (qtp1318894667-17250) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf [junit4] 2> 1410371 INFO (qtp1318894667-17250) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf [junit4] 2> 1410371 INFO (qtp1879748196-17240) [ ] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms [junit4] 2> 1410371 INFO (qtp1879748196-17240) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms. [junit4] 2> 1410371 INFO (qtp1318894667-17250) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms. [junit4] 2> 1410372 INFO (searcherExecutor-7493-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_n1] Registered new searcher Searcher@2c5fb61a[pull_replica_test_kill_leader_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 1410372 INFO (qtp1318894667-17250) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584202944288391168 [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue. [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:49208/solr/pull_replica_test_kill_leader_shard1_replica_n1/ [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.SyncStrategy http://127.0.0.1:49208/solr/pull_replica_test_kill_leader_shard1_replica_n1/ has no replicas [junit4] 2> 1410379 INFO (qtp1318894667-17250) [ ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR [junit4] 2> 1410384 INFO (qtp1318894667-17250) [ ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:49208/solr/pull_replica_test_kill_leader_shard1_replica_n1/ shard1 [junit4] 2> 1410493 INFO (zkCallback-3290-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410493 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410493 INFO (zkCallback-3291-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410493 INFO (zkCallback-3290-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410550 INFO (qtp1318894667-17250) [ ] o.a.s.c.ZkController I am the leader, no recovery necessary [junit4] 2> 1410553 INFO (qtp1318894667-17250) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_leader_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_kill_leader&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1277 [junit4] 2> 1410664 INFO (zkCallback-3290-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410664 INFO (zkCallback-3291-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410664 INFO (zkCallback-3290-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1410664 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1411375 INFO (qtp1879748196-17240) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=pull_replica_test_kill_leader_shard1_replica_p2&action=CREATE&numShards=1&collection=pull_replica_test_kill_leader&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2099 [junit4] 2> 1411380 INFO (qtp1318894667-17245) [ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas [junit4] 2> 1411476 INFO (zkCallback-3290-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1411477 INFO (zkCallback-3290-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1411477 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1411477 INFO (zkCallback-3291-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [2]) [junit4] 2> 1411865 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:49208/solr/pull_replica_test_kill_leader_shard1_replica_n1/ [junit4] 2> 1411867 INFO (qtp1318894667-17253) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0 [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Master's generation: 1 [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Master's version: 0 [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Slave's generation: 1 [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Slave's version: 0 [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher New index in Master. Deleting mine... [junit4] 2> 1411867 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false} [junit4] 2> 1411868 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6ad78170 commitCommandVersion:0 [junit4] 2> 1411868 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4c81f1c7[pull_replica_test_kill_leader_shard1_replica_p2] main] [junit4] 2> 1411868 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush [junit4] 2> 1411869 INFO (searcherExecutor-7492-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_p2] Registered new searcher Searcher@4c81f1c7[pull_replica_test_kill_leader_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())} [junit4] 2> 1412390 INFO (qtp1318894667-17245) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=1&replicationFactor=1&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_kill_leader&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3451 [junit4] 2> 1412400 INFO (qtp1318894667-17241) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1584202946412806144)]} 0 2 [junit4] 2> 1412401 INFO (qtp1318894667-17244) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584202946415951872,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false} [junit4] 2> 1412401 INFO (qtp1318894667-17244) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@46258a0a commitCommandVersion:1584202946415951872 [junit4] 2> 1412407 INFO (qtp1318894667-17244) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@75cc82cf[pull_replica_test_kill_leader_shard1_replica_n1] main] [junit4] 2> 1412408 INFO (qtp1318894667-17244) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush [junit4] 2> 1412410 INFO (searcherExecutor-7493-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_n1] Registered new searcher Searcher@75cc82cf[pull_replica_test_kill_leader_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))} [junit4] 2> 1412411 INFO (qtp1318894667-17244) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=pull_replica_test_kill_leader:5&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 9 [junit4] 2> 1412415 INFO (qtp1318894667-17250) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0 [junit4] 2> 1412419 INFO (qtp1879748196-17246) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412521 INFO (qtp1879748196-17247) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412623 INFO (qtp1879748196-17249) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412726 INFO (qtp1879748196-17251) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412835 INFO (qtp1879748196-17238) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412946 INFO (qtp1879748196-17240) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1412960 INFO (OverseerCollectionConfigSetProcessor-99012683898355717-127.0.0.1:49208_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper [junit4] 2> 1413058 INFO (qtp1879748196-17243) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413168 INFO (qtp1879748196-17246) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413279 INFO (qtp1879748196-17247) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413390 INFO (qtp1879748196-17249) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413499 INFO (qtp1879748196-17251) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413600 INFO (qtp1879748196-17238) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413711 INFO (qtp1879748196-17240) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413820 INFO (qtp1879748196-17243) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1413932 INFO (qtp1879748196-17246) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414043 INFO (qtp1879748196-17247) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414154 INFO (qtp1879748196-17249) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414259 INFO (qtp1879748196-17251) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414363 INFO (qtp1879748196-17238) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414469 INFO (qtp1879748196-17240) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414574 INFO (qtp1879748196-17243) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414675 INFO (qtp1879748196-17243) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414786 INFO (qtp1879748196-17247) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0 [junit4] 2> 1414856 INFO (qtp1318894667-17253) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0 [junit4] 2> 1414856 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Master's generation: 2 [junit4] 2> 1414856 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Master's version: 1510813661972 [junit4] 2> 1414856 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Slave's generation: 2 [junit4] 2> 1414856 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Slave's version: 1510813661439 [junit4] 2> 1414856 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Starting replication process [junit4] 2> 1414857 INFO (qtp1318894667-17241) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=0 [junit4] 2> 1414857 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Number of files in latest index in master: 20 [junit4] 2> 1414858 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Starting download (fullCopy=true) to MockDirectoryWrapper(RAMDirectory@2f0eefd9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4f8bcb68) [junit4] 2> 1414858 INFO (qtp1318894667-17244) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414860 INFO (qtp1318894667-17248) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tib&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414861 INFO (qtp1318894667-17250) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414861 INFO (qtp1318894667-17252) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414862 INFO (qtp1318894667-17245) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414863 INFO (qtp1318894667-17253) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414864 INFO (qtp1318894667-17241) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414864 INFO (qtp1318894667-17244) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414865 INFO (qtp1318894667-17248) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_LuceneFixedGap_0.tii&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414866 INFO (qtp1318894667-17250) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_TestBloomFilteredLucenePostings_0.blm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414867 INFO (qtp1318894667-17252) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.dii&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414867 INFO (qtp1318894667-17245) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_TestBloomFilteredLucenePostings_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414868 INFO (qtp1318894667-17253) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_TestBloomFilteredLucenePostings_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414869 INFO (qtp1318894667-17241) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414869 INFO (qtp1318894667-17244) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_TestBloomFilteredLucenePostings_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414870 INFO (qtp1318894667-17248) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_TestBloomFilteredLucenePostings_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414871 INFO (qtp1318894667-17250) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.dim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414871 INFO (qtp1318894667-17252) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414872 INFO (qtp1318894667-17245) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414872 WARN (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher File segments_2 did not match. expected checksum is 1223811150 and actual is checksum 2579291978. expected length is 200 and actual length is 117 [junit4] 2> 1414873 INFO (qtp1318894667-17253) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_n1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0 [junit4] 2> 1414873 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=true,bytesDownloaded=3260) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@2f0eefd9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4f8bcb68) [junit4] 2> 1414873 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.c.SolrCore Updating index properties... index=index.20171116062744429 [junit4] 2> 1414874 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher removing old index directory MockDirectoryWrapper(RAMDirectory@44f42dbd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bcbcbec) [junit4] 2> 1414874 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used. [junit4] 2> 1414875 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@b8a3dec[pull_replica_test_kill_leader_shard1_replica_p2] main] [junit4] 2> 1414876 INFO (searcherExecutor-7492-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_p2] Registered new searcher Searcher@b8a3dec[pull_replica_test_kill_leader_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))} [junit4] 2> 1414887 INFO (qtp1879748196-17249) [ ] o.a.s.c.S.Request [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=1 status=0 QTime=0 [junit4] 2> 1414887 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.ChaosMonkey monkey: kill jetty! 49208 [junit4] 2> 1414888 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@7705d{HTTP/1.1,[http/1.1]}{127.0.0.1:0} [junit4] 2> 1414888 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1351874805 [junit4] 2> 1414889 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null [junit4] 2> 1414889 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3406ab91: rootName = solr_49208, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@37686bd3 [junit4] 2> 1414895 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null [junit4] 2> 1414895 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7c2470d8: rootName = solr_49208, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@184fd9de [junit4] 2> 1414900 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null [junit4] 2> 1414900 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@56869072: rootName = solr_49208, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@7175812c [junit4] 2> 1414902 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:49208_solr [junit4] 2> 1414903 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null [junit4] 2> 1414903 INFO (zkCallback-3290-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 1414903 INFO (zkCallback-3298-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 1414903 INFO (coreCloseExecutor-7504-thread-1) [ ] o.a.s.c.SolrCore [pull_replica_test_kill_leader_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@29e5cb3e [junit4] 2> 1414903 INFO (zkCallback-3291-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 1414903 INFO (coreCloseExecutor-7504-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_kill_leader.shard1.replica_n1, tag=702925630 [junit4] 2> 1414903 INFO (coreCloseExecutor-7504-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@58139f1e: rootName = solr_49208, domain = solr.core.pull_replica_test_kill_leader.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.pull_replica_test_kill_leader.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3403ee2f [junit4] 2> 1414904 INFO (zkCallback-3290-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [1]) [junit4] 2> 1414904 INFO (zkCallback-3290-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [1]) [junit4] 2> 1414904 INFO (zkCallback-3291-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [1]) [junit4] 2> 1414904 INFO (zkCallback-3291-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_kill_leader/state.json] for collection [pull_replica_test_kill_leader] has occurred - updating... (live nodes size: [1]) [junit4] 2> 1414910 INFO (coreCloseExecutor-7504-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_kill_leader.shard1.leader, tag=702925630 [junit4] 2> 1414911 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.Overseer Overseer (id=99012683898355717-127.0.0.1:49208_solr-n_0000000000) closing [junit4] 2> 1414912 INFO (OverseerStateUpdate-99012683898355717-127.0.0.1:49208_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49208_solr [junit4] 2> 1414912 WARN (OverseerAutoScalingTriggerThread-99012683898355717-127.0.0.1:49208_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting. [junit4] 2> 1414914 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5e1c0760{/solr,null,UNAVAILABLE} [junit4] 2> 1414914 INFO (zkCallback-3291-thread-3) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49207_solr [junit4] 2> 1414915 INFO (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 49208 [junit4] 2> 1414916 INFO (zkCallback-3291-thread-3) [ ] o.a.s.c.Overseer Overseer (id=99012683898355718-127.0.0.1:49207_solr-n_0000000001) starting [junit4] 2> 1414919 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (0) java.net.SocketException: Connection reset, retry? 0 [junit4] 2> 1418937 INFO (qtp1879748196-17243) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4016 [junit4] 2> 1418938 INFO (qtp1879748196-17243) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1418939 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 1 [junit4] 2> 1421904 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1421904 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1422960 INFO (qtp1879748196-17247) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4018 [junit4] 2> 1422960 INFO (qtp1879748196-17247) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1422960 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 2 [junit4] 2> 1425955 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1425955 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1426986 INFO (qtp1879748196-17249) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4024 [junit4] 2> 1426987 INFO (qtp1879748196-17249) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1426988 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 3 [junit4] 2> 1429976 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1429977 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1431050 INFO (qtp1879748196-17238) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4060 [junit4] 2> 1431051 INFO (qtp1879748196-17238) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1431052 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 4 [junit4] 2> 1433981 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1433981 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1435082 INFO (qtp1879748196-17251) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4027 [junit4] 2> 1435082 INFO (qtp1879748196-17251) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1435083 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 5 [junit4] 2> 1438026 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1438026 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1439112 INFO (qtp1879748196-17246) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4024 [junit4] 2> 1439112 INFO (qtp1879748196-17246) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1439113 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 0 [junit4] 2> 1442052 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1442052 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1443129 INFO (qtp1879748196-17240) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4013 [junit4] 2> 1443129 INFO (qtp1879748196-17240) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1443130 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [junit4] 2> "autoAddReplicas":"false", [junit4] 2> "nrtReplicas":"1", [junit4] 2> "tlogReplicas":"0"} with live_nodes=[127.0.0.1:49207_solr], retry? 1 [junit4] 2> 1445130 INFO (ScheduledTrigger-7507-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging. [junit4] 2> 1445136 INFO (qtp1879748196-17247) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=1 [junit4] 2> 1445138 INFO (AutoscalingActionExecutor-7508-thread-1) [ ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: { [junit4] 2> "id":"14f77d05f244e2c0T8pkl4kzzyd3wp5djs093k3fzf", [junit4] 2> "source":".auto_add_replicas", [junit4] 2> "eventTime":1510813664499000000, [junit4] 2> "eventType":"NODELOST", [junit4] 2> "properties":{ [junit4] 2> "eventTimes":[1510813664499000000], [junit4] 2> "_enqueue_time_":1510813694699000000, [junit4] 2> "nodeNames":["127.0.0.1:49208_solr"]}} [junit4] 2> 1446098 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.ReplicationHandler Ignoring exception matching No registered leader was found [junit4] 2> 1446098 INFO (indexFetcher-7502-thread-1) [ ] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication [junit4] 2> 1447155 INFO (qtp1879748196-17243) [ ] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_kill_leader_shard1_replica_p2] webapp=/solr path=/update params={wt=javabin&version=2}{} 0 4022 [junit4] 2> 1447155 INFO (qtp1879748196-17243) [ ] o.a.s.h.RequestHandlerBase Ignoring exception matching No registered leader was found [junit4] 2> 1447156 ERROR (TEST-TestPullReplica.testKillLeader-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection [pull_replica_test_kill_leader] failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49207/solr/pull_replica_test_kill_leader_shard1_replica_p2: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_kill_leader slice: shard1 saw state=DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={ [junit4] 2> "pullReplicas":"1", [junit4] 2> "replicationFactor":"1", [junit4] 2> "shards":{"shard1":{ [junit4] 2> "range":"80000000-7fffffff", [junit4] 2> "state":"active", [junit4] 2> "replicas":{ [junit4] 2> "core_node3":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_n1", [junit4] 2> "base_url":"http://127.0.0.1:49208/solr", [junit4] 2> "node_name":"127.0.0.1:49208_solr", [junit4] 2> "state":"down", [junit4] 2> "type":"NRT", [junit4] 2> "leader":"true"}, [junit4] 2> "core_node4":{ [junit4] 2> "core":"pull_replica_test_kill_leader_shard1_replica_p2", [junit4] 2> "base_url":"http://127.0.0.1:49207/solr", [junit4] 2> "node_name":"127.0.0.1:49207_solr", [junit4] 2> "state":"active", [junit4] 2> "type":"PULL"}}}}, [junit4] 2> "router":{"name":"compositeId"}, [junit4] 2> "maxShardsPerNode":"100", [ [...truncated too long message...] estEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@254a5af6: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2fa9be00 [junit4] 2> 1734001 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null [junit4] 2> 1734001 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@9bcc0c5: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@17ffe6c3 [junit4] 2> 1734002 INFO (coreCloseExecutor-7593-thread-1) [ ] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@806ed23 [junit4] 2> 1734002 INFO (coreCloseExecutor-7593-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=134671651 [junit4] 2> 1734002 INFO (coreCloseExecutor-7593-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2fd1a50d: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@5308e8a7 [junit4] 2> 1734006 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1b2e941d{/solr,null,UNAVAILABLE} [junit4] 2> 1734008 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@75afee0{HTTP/1.1,[http/1.1]}{127.0.0.1:0} [junit4] 2> 1734008 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1994025384 [junit4] 2> 1734008 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null [junit4] 2> 1734008 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@393c3a6b: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@21f7992f [junit4] 2> 1734014 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null [junit4] 2> 1734014 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6a0ca1b0: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2fa9be00 [junit4] 2> 1734018 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null [junit4] 2> 1734018 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@122ce6de: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@17ffe6c3 [junit4] 2> 1734018 INFO (coreCloseExecutor-7595-thread-1) [ ] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@50273a76 [junit4] 2> 1734018 INFO (coreCloseExecutor-7595-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1344748150 [junit4] 2> 1734018 INFO (coreCloseExecutor-7595-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@21656e49: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@431656c [junit4] 2> 1734024 INFO (TEST-TestReplicationHandler.testEmptyCommits-seed#[E66D3DABF20F30F0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@263a5c60{/solr,null,UNAVAILABLE} [junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_E66D3DABF20F30F0-001 [junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {name=PostingsFormat(name=Asserting), id=PostingsFormat(name=Asserting), newname=PostingsFormat(name=Asserting)}, docValues:{}, maxPointsInLeafNode=801, maxMBSortInHeap=7.944338023809664, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@686f7056), locale=vun, timezone=Asia/Anadyr [junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 9 (64-bit)/cpus=3,threads=1,free=237980960,total=490209280 [junit4] 2> NOTE: All tests run in this JVM: [BlobRepositoryCloudTest, UpdateRequestProcessorFactoryTest, PolyFieldTest, TestAddFieldRealTimeGet, TestExpandComponent, SubstringBytesRefFilterTest, JsonLoaderTest, TestPerFieldSimilarity, ClassificationUpdateProcessorTest, AnalysisAfterCoreReloadTest, TestInfoStreamLogging, TestDocBasedVersionConstraints, TestSegmentSorting, TestXmlQParserPlugin, ParsingFieldUpdateProcessorsTest, TestQueryUtils, TestComponentsName, LeaderElectionContextKeyTest, MigrateRouteKeyTest, TestPayloadCheckQParserPlugin, PingRequestHandlerTest, SolrGangliaReporterTest, TestNestedDocsSort, DeleteReplicaTest, TestStandardQParsers, HealthCheckHandlerTest, CopyFieldTest, TestQueryWrapperFilter, DocValuesNotIndexedTest, TestTlogReplica, TestReload, TestHdfsCloudBackupRestore, CdcrBootstrapTest, TestPKIAuthenticationPlugin, IndexSchemaRuntimeFieldTest, TestMiniSolrCloudClusterSSL, PeerSyncReplicationTest, FileUtilsTest, OpenExchangeRatesOrgProviderTest, AddBlockUpdateTest, TestTrie, TestHdfsBackupRestoreCore, TestWordDelimiterFilterFactory, UUIDUpdateProcessorFallbackTest, RequestLoggingTest, SampleTest, BlockCacheTest, TestPivotHelperCode, ConcurrentDeleteAndCreateCollectionTest, TestReplicaProperties, CollectionsAPIAsyncDistributedZkTest, TestFastLRUCache, TestJsonFacets, TestCoreAdminApis, TestUnifiedSolrHighlighter, AnalyticsQueryTest, CoreAdminCreateDiscoverTest, TestFastOutputStream, TestPointFields, BaseCdcrDistributedZkTest, TestDistribIDF, TestSolrQueryResponse, MetricsHandlerTest, LeaderInitiatedRecoveryOnShardRestartTest, ZkStateWriterTest, TestManagedSchemaAPI, TestDistributedMissingSort, TestFieldCacheVsDocValues, OverseerStatusTest, TestObjectReleaseTracker, DistributedExpandComponentTest, TestNumericTokenStream, TestSystemIdResolver, CustomCollectionTest, QueryElevationComponentTest, HdfsLockFactoryTest, TestConfigSetProperties, BadComponentTest, ConnectionManagerTest, CoreSorterTest, SimplePostToolTest, TestDynamicFieldCollectionResource, TestClassicSimilarityFactory, JavabinLoaderTest, NumberUtilsTest, DistributedFacetPivotWhiteBoxTest, CoreAdminHandlerTest, TestOrdValues, PluginInfoTest, NodeMutatorTest, TestLegacyFieldReuse, TestJettySolrRunner, ResponseHeaderTest, TestZkAclsWithHadoopAuth, TestMacroExpander, TestCloudPseudoReturnFields, TestUseDocValuesAsStored2, TestSolrCloudWithSecureImpersonation, UtilsToolTest, TestLegacyFieldCache, TestBM25SimilarityFactory, TestRandomRequestDistribution, PeerSyncWithIndexFingerprintCachingTest, TestCollationField, FullSolrCloudDistribCmdsTest, TestZkChroot, TestRawTransformer, TestSearcherReuse, TestExactStatsCache, CacheHeaderTest, TestSizeLimitedDistributedMap, AssignTest, TestNamedUpdateProcessors, TestScoreJoinQPScore, BlockJoinFacetRandomTest, TestSolrCloudSnapshots, TestCloudJSONFacetJoinDomain, TestLuceneMatchVersion, DistributedFacetPivotLargeTest, ShardRoutingCustomTest, TestBulkSchemaAPI, BasicDistributedZk2Test, RollingRestartTest, TestRecoveryHdfs, TestPartialUpdateDeduplication, StatelessScriptUpdateProcessorFactoryTest, LargeFieldTest, TestSchemalessBufferedUpdates, TestPostingsSolrHighlighter, SystemInfoHandlerTest, DocValuesTest, TestBlendedInfixSuggestions, ZkSolrClientTest, TestGraphMLResponseWriter, BooleanFieldTest, BasicZkTest, SharedFSAutoReplicaFailoverTest, TestBulkSchemaConcurrent, V2StandaloneTest, TestCoreContainer, SolrIndexSplitterTest, ChaosMonkeySafeLeaderTest, TestIndexSearcher, TestFunctionQuery, TestFieldCache, TestDistributedGrouping, ZkFailoverTest, SynonymTokenizerTest, ExternalFileFieldSortTest, LukeRequestHandlerTest, TestStressInPlaceUpdates, RuleEngineTest, TestQueryTypes, TestGroupingSearch, DistributedFacetPivotSmallAdvancedTest, TestSolrDeletionPolicy1, SearchHandlerTest, TestNoOpRegenerator, ExitableDirectoryReaderTest, MoveReplicaTest, TestCloudDeleteByQuery, TestSubQueryTransformerDistrib, TestFieldCacheSort, ReplicaListTransformerTest, TestConfigSetsAPI, TestTrieFacet, DistributedIntervalFacetingTest, FullHLLTest, TestNumericRangeQuery32, DebugComponentTest, ReplicationFactorTest, SolrJmxReporterCloudTest, TestValueSourceCache, TestLegacyTerms, RankQueryTest, DistributedQueueTest, AliasIntegrationTest, TestConfig, TestMissingGroups, CheckHdfsIndexTest, SolrCoreCheckLockOnStartupTest, SpellPossibilityIteratorTest, SecurityConfHandlerTest, TestDocumentBuilder, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, ShardRoutingTest, TestReplicationHandler] [junit4] Completed [384/749 (2!)] on J1 in 86.66s, 16 tests, 1 error, 1 skipped <<< FAILURES! [...truncated 41351 lines...]
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org