Build: https://builds.apache.org/job/Lucene-Solr-Tests-6.x/8/
1 tests failed. FAILED: org.apache.solr.cloud.OverseerTest.testExternalClusterStateChangeBehavior Error Message: Illegal state, was: down expected:active clusterState:live nodes:[]collections:{c1=DocCollection(c1)={ "shards":{"shard1":{ "parent":null, "range":null, "state":"active", "replicas":{"core_node1":{ "base_url":"http://127.0.0.1/solr", "node_name":"node1", "core":"core1", "roles":"", "state":"down"}}}}, "router":{"name":"implicit"}}, test=LazyCollectionRef(test)} Stack Trace: java.lang.AssertionError: Illegal state, was: down expected:active clusterState:live nodes:[]collections:{c1=DocCollection(c1)={ "shards":{"shard1":{ "parent":null, "range":null, "state":"active", "replicas":{"core_node1":{ "base_url":"http://127.0.0.1/solr", "node_name":"node1", "core":"core1", "roles":"", "state":"down"}}}}, "router":{"name":"implicit"}}, test=LazyCollectionRef(test)} at __randomizedtesting.SeedInfo.seed([3423860D0A8E2EF6:5C3D85E1E81E74B8]:0) at org.junit.Assert.fail(Assert.java:93) at org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus(AbstractDistribZkTestBase.java:236) at org.apache.solr.cloud.OverseerTest.testExternalClusterStateChangeBehavior(OverseerTest.java:1273) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) at java.lang.Thread.run(Thread.java:745) Build Log: [...truncated 10751 lines...] [junit4] Suite: org.apache.solr.cloud.OverseerTest [junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerTest_3423860D0A8E2EF6-001/init-core-data-001 [junit4] 2> 190186 INFO (SUITE-OverseerTest-seed#[3423860D0A8E2EF6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) [junit4] 2> 190186 INFO (SUITE-OverseerTest-seed#[3423860D0A8E2EF6]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore [junit4] 2> 190187 INFO (SUITE-OverseerTest-seed#[3423860D0A8E2EF6]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end [junit4] 2> 190209 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testPlaceholders [junit4] 2> 190209 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 190209 INFO (Thread-726) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 190209 INFO (Thread-726) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 190309 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer start zk server on port:43955 [junit4] 2> 190310 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 190310 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 190314 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@67ba1da name:ZooKeeperConnection Watcher:127.0.0.1:43955/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 190314 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 190314 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 190314 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 190316 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 190322 INFO (zkCallback-166-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@25e31ccd name:ZooKeeperConnection Watcher:127.0.0.1:43955 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 190322 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 190322 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 190331 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 190332 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 190336 INFO (zkCallback-167-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4193c11f name:ZooKeeperConnection Watcher:127.0.0.1:43955 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 190336 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 190337 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 190337 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 190342 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 190343 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 190344 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 190346 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json [junit4] 2> 190347 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json [junit4] 2> 190348 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 190349 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 190350 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 190351 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 190352 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4b8a72ca name:ZooKeeperConnection Watcher:127.0.0.1:43955/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 190352 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 190353 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 190354 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 190355 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 190356 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1 [junit4] 2> 190357 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 190357 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 190358 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 190357 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 190358 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 190359 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 190360 INFO (zkCallback-169-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@69e7fcf4 name:ZooKeeperConnection Watcher:127.0.0.1:43955/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 190360 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 190360 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 190360 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true [junit4] 2> 190366 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect [junit4] 2> 190367 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election [junit4] 2> 190368 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 190369 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95497788559720452-127.0.0.1:43955_solr-n_0000000000 [junit4] 2> 190369 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43955_solr [junit4] 2> 190369 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader [junit4] 2> 190370 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497788559720452-127.0.0.1:43955_solr-n_0000000000) starting [junit4] 2> 190371 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 190372 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work [junit4] 2> 190374 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 190375 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 190376 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 190378 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 190380 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000 [junit4] 2> 190380 INFO (OverseerCollectionConfigSetProcessor-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations [junit4] 2> 190381 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue [junit4] 2> 190382 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 190383 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 190385 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 0, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"recovering", [junit4] 2> "node_name":"node1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"12", [junit4] 2> "base_url":"http://node1/solr/"} current state version: 0 [junit4] 2> 190385 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=12 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"recovering", [junit4] 2> "node_name":"node1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"12", [junit4] 2> "base_url":"http://node1/solr/"} [junit4] 2> 190385 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1 [junit4] 2> 190386 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2 [junit4] 2> 190387 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 190387 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 190389 INFO (zkCallback-169-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 190882 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard2/election [junit4] 2> 190884 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 190884 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 190884 INFO (zkCallback-169-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 190888 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard2/election/95497788559720451-node1_core1-n_0000000000 [junit4] 2> 190888 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard2 [junit4] 2> 190891 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard2/leader after winning as /collections/collection1/leader_elect/shard2/election/95497788559720451-node1_core1-n_0000000000 [junit4] 2> 190894 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard2", [junit4] 2> "collection":"collection1", [junit4] 2> "base_url":"http://node1/solr/", [junit4] 2> "core":"core1", [junit4] 2> "state":"active"} current state version: 1 [junit4] 2> 190894 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1) [junit4] 2> 190895 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard2/election/95497788559720451-node1_core1-n_0000000000 [junit4] 2> 190896 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection1/leaders/shard2/leader 1 [junit4] 2> 190897 INFO (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 190897 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 190898 INFO (zkCallback-165-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 192385 INFO (OverseerCollectionConfigSetProcessor-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:358) [junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:175) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 192385 INFO (OverseerCollectionConfigSetProcessor-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=95497788559720452-127.0.0.1:43955_solr-n_0000000000) am no longer a leader. [junit4] 2> 192394 WARN (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.Overseer Solr cannot talk to ZK, exiting Overseer main queue loop [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-0000000001 [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.cloud.DistributedQueue.removeFirst(DistributedQueue.java:383) [junit4] 2> at org.apache.solr.cloud.DistributedQueue.poll(DistributedQueue.java:186) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:231) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 192395 INFO (OverseerStateUpdate-95497788559720452-127.0.0.1:43955_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43955_solr [junit4] 2> 192397 WARN (zkCallback-168-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 192398 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267) [junit4] 2> 192399 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43955 43955 [junit4] 2> 192503 INFO (Thread-726) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43955 43955 [junit4] 2> 192503 WARN (Thread-726) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 3 /solr/aliases.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 3 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 3 /solr/live_nodes [junit4] 2> 3 /solr/collections [junit4] 2> [junit4] 2> 192504 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testPlaceholders [junit4] 2> 192504 INFO (TEST-OverseerTest.testPlaceholders-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497788559720452-127.0.0.1:43955_solr-n_0000000000) closing [junit4] 2> 192506 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBadQueueItem [junit4] 2> 192507 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 192507 INFO (Thread-727) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 192507 INFO (Thread-727) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 192607 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer start zk server on port:44797 [junit4] 2> 192608 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 192609 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 192614 INFO (zkCallback-172-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2ead0305 name:ZooKeeperConnection Watcher:127.0.0.1:44797 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 192615 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 192615 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 192618 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 192618 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 192620 INFO (zkCallback-173-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7a4af6cd name:ZooKeeperConnection Watcher:127.0.0.1:44797 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 192620 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 192620 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 192620 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 192623 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 192623 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 192625 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5e25b053 name:ZooKeeperConnection Watcher:127.0.0.1:44797/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 192625 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 192625 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 192627 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 192629 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 192631 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 192632 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json [junit4] 2> 192633 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json [junit4] 2> 192634 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 192635 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 192639 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@697edc63 name:ZooKeeperConnection Watcher:127.0.0.1:44797/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 192639 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 192639 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 192639 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true [junit4] 2> 192644 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect [junit4] 2> 192646 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election [junit4] 2> 192647 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 192648 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95497788710518787-127.0.0.1:44797_solr-n_0000000000 [junit4] 2> 192648 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44797_solr [junit4] 2> 192648 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader [junit4] 2> 192650 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497788710518787-127.0.0.1:44797_solr-n_0000000000) starting [junit4] 2> 192651 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 192653 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work [junit4] 2> 192656 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 192658 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 192661 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 192664 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 192667 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000 [junit4] 2> 192668 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 192668 INFO (OverseerCollectionConfigSetProcessor-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations [junit4] 2> 192668 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue [junit4] 2> 192669 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 192669 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 192670 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 192670 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 192670 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 192674 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@26e2aa5d name:ZooKeeperConnection Watcher:127.0.0.1:44797/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 192674 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 192674 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 192676 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 192678 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 192678 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1 [junit4] 2> 192680 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 192680 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 192680 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 192680 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 192681 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 192681 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 192683 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 0 [junit4] 2> 192684 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 192684 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1 [junit4] 2> 192684 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2 [junit4] 2> 192686 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 192686 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 192686 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193183 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard2/election [junit4] 2> 193185 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 193185 INFO (zkCallback-175-thread-2) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 193185 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 193191 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 193192 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard2 [junit4] 2> 193195 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard2/leader after winning as /collections/collection1/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 193205 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard2", [junit4] 2> "collection":"collection1", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core1", [junit4] 2> "state":"active"} current state version: 1 [junit4] 2> 193207 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core2", [junit4] 2> "core_node_name":"node2", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 1 [junit4] 2> 193207 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core2", [junit4] 2> "core_node_name":"node2", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 193207 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3 [junit4] 2> 193207 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard3 [junit4] 2> 193310 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193311 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193311 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193706 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard3/election [junit4] 2> 193711 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 193712 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard3 [junit4] 2> 193714 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard3/leader after winning as /collections/collection1/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 193717 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard3", [junit4] 2> "collection":"collection1", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core2", [junit4] 2> "state":"active"} current state version: 2 [junit4] 2> 193719 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core3", [junit4] 2> "core_node_name":"node3", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 2 [junit4] 2> 193720 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core3", [junit4] 2> "core_node_name":"node3", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 193720 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3 [junit4] 2> 193720 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 193823 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193823 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 193823 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194219 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election [junit4] 2> 194225 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 194226 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1 [junit4] 2> 194229 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 194232 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"collection1", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core3", [junit4] 2> "state":"active"} current state version: 3 [junit4] 2> 194335 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194335 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194335 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194386 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core0", [junit4] 2> "core_node_name":"node0", [junit4] 2> "collection":"", [junit4] 2> "numShards":"1", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 4 [junit4] 2> 194387 ERROR (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.CollectionMutator Skipping invalid Overseer message because it has no collection specified: { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core0", [junit4] 2> "core_node_name":"node0", [junit4] 2> "collection":"", [junit4] 2> "numShards":"1", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 194390 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"deletecore", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core0", [junit4] 2> "core_node_name":"node0", [junit4] 2> "collection":""} current state version: 4 [junit4] 2> 194390 ERROR (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.CollectionMutator Skipping invalid Overseer message because it has no collection specified: { [junit4] 2> "operation":"deletecore", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core0", [junit4] 2> "core_node_name":"node0", [junit4] 2> "collection":""} [junit4] 2> 194392 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 4 [junit4] 2> 194392 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"node1", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 194393 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection2 [junit4] 2> 194393 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2 [junit4] 2> 194497 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194497 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194497 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 194890 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 194891 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection1/leaders/shard2/leader 1 [junit4] 2> 194892 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leader_elect/shard2/election [junit4] 2> 194893 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 194893 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 194893 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 194897 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection2/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 194898 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leaders/shard2 [junit4] 2> 194901 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection2/leaders/shard2/leader after winning as /collections/collection2/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 194904 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard2", [junit4] 2> "collection":"collection2", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core1", [junit4] 2> "state":"active"} current state version: 5 [junit4] 2> 194907 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core2", [junit4] 2> "core_node_name":"node2", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 5 [junit4] 2> 194907 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core2", [junit4] 2> "core_node_name":"node2", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 194907 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3 [junit4] 2> 194907 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard3 [junit4] 2> 195010 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195010 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195011 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195406 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 195407 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection1/leaders/shard3/leader 1 [junit4] 2> 195408 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leader_elect/shard3/election [junit4] 2> 195416 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection2/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 195417 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leaders/shard3 [junit4] 2> 195419 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection2/leaders/shard3/leader after winning as /collections/collection2/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 195428 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard3", [junit4] 2> "collection":"collection2", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core2", [junit4] 2> "state":"active"} current state version: 6 [junit4] 2> 195430 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core3", [junit4] 2> "core_node_name":"node3", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 6 [junit4] 2> 195430 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"active", [junit4] 2> "node_name":"127.0.0.1", [junit4] 2> "core":"core3", [junit4] 2> "core_node_name":"node3", [junit4] 2> "collection":"collection2", [junit4] 2> "numShards":"3", [junit4] 2> "base_url":"http://127.0.0.1/solr/"} [junit4] 2> 195430 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3 [junit4] 2> 195430 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 195533 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195534 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195534 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 195929 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 195930 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection1/leaders/shard1/leader 1 [junit4] 2> 195931 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leader_elect/shard1/election [junit4] 2> 195936 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection2/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 195938 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leaders/shard1 [junit4] 2> 195941 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection2/leaders/shard1/leader after winning as /collections/collection2/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 195946 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"collection2", [junit4] 2> "base_url":"http://127.0.0.1/solr/", [junit4] 2> "core":"core3", [junit4] 2> "state":"active"} current state version: 7 [junit4] 2> 196052 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 196052 INFO (zkCallback-174-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 196052 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 196097 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection2/leader_elect/shard1/election/95497788710518788-127.0.0.1_core3-n_0000000000 [junit4] 2> 196099 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection2/leaders/shard1/leader 1 [junit4] 2> 196100 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection2/leader_elect/shard3/election/95497788710518788-127.0.0.1_core2-n_0000000000 [junit4] 2> 196100 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection2/leaders/shard3/leader 1 [junit4] 2> 196101 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection2/leader_elect/shard2/election/95497788710518788-127.0.0.1_core1-n_0000000000 [junit4] 2> 196101 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection2/leaders/shard2/leader 1 [junit4] 2> 196102 INFO (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 196102 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 196103 INFO (zkCallback-175-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 197602 WARN (zkCallback-178-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 197604 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44797 44797 [junit4] 2> 197776 INFO (Thread-727) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44797 44797 [junit4] 2> 197777 WARN (Thread-727) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 3 /solr/aliases.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 3 /solr/clusterstate.json [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 3 /solr/live_nodes [junit4] 2> 3 /solr/collections [junit4] 2> [junit4] 2> 197777 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testBadQueueItem [junit4] 2> 197777 INFO (TEST-OverseerTest.testBadQueueItem-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497788710518787-127.0.0.1:44797_solr-n_0000000000) closing [junit4] 2> 197777 INFO (OverseerStateUpdate-95497788710518787-127.0.0.1:44797_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44797_solr [junit4] 2> 197778 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267) [junit4] 2> 197780 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDoubleAssignment [junit4] 2> 197780 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER [junit4] 2> 197781 INFO (Thread-728) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 197781 INFO (Thread-728) [ ] o.a.s.c.ZkTestServer Starting server [junit4] 2> 197881 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer start zk server on port:41154 [junit4] 2> 197881 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 197882 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 197890 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@88fb0c2 name:ZooKeeperConnection Watcher:127.0.0.1:41154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 197890 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 197891 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 197891 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 197891 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 197892 INFO (zkCallback-180-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@54aad0af name:ZooKeeperConnection Watcher:127.0.0.1:41154 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 197893 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 197893 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 197894 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 197895 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 197896 INFO (zkCallback-181-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4b75280 name:ZooKeeperConnection Watcher:127.0.0.1:41154 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 197896 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 197896 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 197896 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr [junit4] 2> 197907 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes [junit4] 2> 197909 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections [junit4] 2> 197910 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json [junit4] 2> 197911 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json [junit4] 2> 197912 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json [junit4] 2> 197913 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 197914 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 197915 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 197916 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 197927 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@55931d48 name:ZooKeeperConnection Watcher:127.0.0.1:41154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 197927 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 197927 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 197929 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 197930 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 197931 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1 [junit4] 2> 197932 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 197932 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0]) [junit4] 2> 197932 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 197934 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 197948 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 197949 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 197952 INFO (zkCallback-183-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@443acd84 name:ZooKeeperConnection Watcher:127.0.0.1:41154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 197952 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 197953 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 197953 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true [junit4] 2> 197956 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect [junit4] 2> 197958 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election [junit4] 2> 197959 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=null) closing [junit4] 2> 197959 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95497789055893508-127.0.0.1:41154_solr-n_0000000000 [junit4] 2> 197959 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41154_solr [junit4] 2> 197959 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader [junit4] 2> 197960 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497789055893508-127.0.0.1:41154_solr-n_0000000000) starting [junit4] 2> 197961 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue [junit4] 2> 197963 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work [junit4] 2> 197964 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure [junit4] 2> 197966 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running [junit4] 2> 197968 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed [junit4] 2> 197972 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work [junit4] 2> 197977 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000 [junit4] 2> 197978 INFO (OverseerCollectionConfigSetProcessor-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations [junit4] 2> 197978 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue [junit4] 2> 197981 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 197989 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1) [junit4] 2> 197991 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"state", [junit4] 2> "state":"recovering", [junit4] 2> "node_name":"node1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"core_node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"1", [junit4] 2> "base_url":"http://node1/solr/"} current state version: 0 [junit4] 2> 197992 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={ [junit4] 2> "operation":"state", [junit4] 2> "state":"recovering", [junit4] 2> "node_name":"node1", [junit4] 2> "core":"core1", [junit4] 2> "core_node_name":"core_node1", [junit4] 2> "collection":"collection1", [junit4] 2> "numShards":"1", [junit4] 2> "base_url":"http://node1/solr/"} [junit4] 2> 197992 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1 [junit4] 2> 197992 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1 [junit4] 2> 197996 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 197996 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 197998 INFO (zkCallback-183-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1]) [junit4] 2> 198481 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election [junit4] 2> 198482 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 198482 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 198482 INFO (zkCallback-183-thread-2) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating... [junit4] 2> 198485 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95497789055893507-node1_core1-n_0000000000 [junit4] 2> 198486 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1 [junit4] 2> 198488 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/95497789055893507-node1_core1-n_0000000000 [junit4] 2> 198493 INFO (OverseerStateUpdate-95497789055893508-127.0.0.1:41154_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = { [junit4] 2> "operation":"leader", [junit4] 2> "shard":"shard1", [junit4] 2> "collection":"collection1", [junit4] 2> "base_url":"http://node1/solr/", [junit4] 2> "core":"core1", [junit4] 2> "state":"active"} current state version: 1 [junit4] 2> 198494 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1) [junit4] 2> 198494 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard1/election/95497789055893507-node1_core1-n_0000000000 [junit4] 2> 198494 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ShardLeaderElectionContextBase Removing leader registration node on cancel: /collections/collection1/leaders/shard1/leader 1 [junit4] 2> 198495 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 198495 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 198495 INFO (zkCallback-183-thread-2) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 198496 INFO (zkCallback-179-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 198496 INFO (zkCallback-183-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 198496 INFO (zkCallback-182-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0) [junit4] 2> 198497 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider [junit4] 2> 198498 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper [junit4] 2> 198499 INFO (zkCallback-186-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4aef6be0 name:ZooKeeperConnection Watcher:127.0.0.1:41154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 198499 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper [junit4] 2> 198499 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider [junit4] 2> 198501 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... [junit4] 2> 198502 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0) [junit4] 2> 198503 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1 [junit4] 2 [...truncated too long message...] A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3]) [junit4] 2> 244319 INFO (zkCallback-301-thread-2) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3]) [junit4] 2> 244320 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2) [junit4] 2> 244321 INFO (zkCallback-303-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2) [junit4] 2> 244321 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception [junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15346bc4acd0013, likely client has closed socket [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228) [junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> 245820 WARN (zkCallback-301-thread-2) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 245821 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2]) [junit4] 2> 245821 INFO (zkCallback-303-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2]) [junit4] 2> 245822 INFO (zkCallback-303-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1) [junit4] 2> 247264 ERROR (OverseerHdfsCoreFailoverThread-95497791627657219-127.0.0.1:60767_solr-n_0000000000) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread OverseerAutoReplicaFailoverThread had an error in its thread work loop.:org.apache.solr.common.SolrException: Error reading cluster properties [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:703) [junit4] 2> at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:152) [junit4] 2> at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131) [junit4] 2> at java.lang.Thread.run(Thread.java:745) [junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterprops.json [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:314) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:311) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:311) [junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getClusterProps(ZkStateReader.java:694) [junit4] 2> ... 3 more [junit4] 2> [junit4] 2> 247321 WARN (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 247322 INFO (zkCallback-303-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1]) [junit4] 2> 248823 WARN (zkCallback-303-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes] [junit4] 2> 248824 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60767 60767 [junit4] 2> 248944 INFO (Thread-738) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60767 60767 [junit4] 2> 248946 WARN (Thread-738) [ ] o.a.s.c.ZkTestServer Watch limit violations: [junit4] 2> Maximum concurrent create/delete watches above limit: [junit4] 2> [junit4] 2> 20 /solr/aliases.json [junit4] 2> [junit4] 2> Maximum concurrent data watches above limit: [junit4] 2> [junit4] 2> 20 /solr/clusterstate.json [junit4] 2> 2 /solr/collections/collection1/leader_elect/shard2/election/95497791627657221-node1_core1-n_0000000000 [junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/95497791627657232-node12_core12-n_0000000000 [junit4] 2> [junit4] 2> Maximum concurrent children watches above limit: [junit4] 2> [junit4] 2> 20 /solr/live_nodes [junit4] 2> 20 /solr/collections [junit4] 2> [junit4] 2> 248946 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[3423860D0A8E2EF6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testShardAssignmentBigger [junit4] 2> 248946 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[3423860D0A8E2EF6]) [ ] o.a.s.c.Overseer Overseer (id=95497791627657219-127.0.0.1:60767_solr-n_0000000000) closing [junit4] 2> 248947 INFO (OverseerStateUpdate-95497791627657219-127.0.0.1:60767_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60767_solr [junit4] 2> 248947 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data [junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127) [junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) [junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89) [junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267) [junit4] 2> 251948 INFO (SUITE-OverseerTest-seed#[3423860D0A8E2EF6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerTest_3423860D0A8E2EF6-001 [junit4] 2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=128, maxMBSortInHeap=4.764527126031389, sim=ClassicSimilarity, locale=da-DK, timezone=Antarctica/Casey [junit4] 2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_66 (64-bit)/cpus=4,threads=1,free=204528192,total=467664896 [junit4] 2> NOTE: All tests run in this JVM: [TestExtendedDismaxParser, HdfsChaosMonkeyNothingIsSafeTest, LukeRequestHandlerTest, TestConfigSets, SharedFSAutoReplicaFailoverTest, TestCloudDeleteByQuery, TestUseDocValuesAsStored2, ClusterStateTest, BlockJoinFacetDistribTest, DateMathParserTest, ZkStateWriterTest, HdfsDirectoryTest, TestSortingResponseWriter, CursorMarkTest, TestUtils, TestSchemaVersionResource, TestJmxMonitoredMap, LeaderInitiatedRecoveryOnCommitTest, DistributedFacetPivotSmallAdvancedTest, TestSolrDynamicMBean, TestCharFilters, CdcrRequestHandlerTest, BigEndianAscendingWordDeserializerTest, TestExactStatsCache, TestStressRecovery, TestSolrQueryParser, TestJmxIntegration, TestReload, TestSolrJ, UpdateParamsTest, ExplicitHLLTest, TestPerFieldSimilarityClassic, TestSolrDeletionPolicy1, TestSchemaNameResource, TestFieldSortValues, OverseerTest] [junit4] Completed [85/583 (1!)] on J2 in 61.78s, 13 tests, 1 failure, 1 skipped <<< FAILURES! [...truncated 1622 lines...] BUILD FAILED /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/build.xml:740: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/build.xml:684: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/build.xml:59: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build.xml:233: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/common-build.xml:524: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/lucene/common-build.xml:1457: The following error occurred while executing this line: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/lucene/common-build.xml:1014: There were test failures: 583 suites (8 ignored), 2352 tests, 1 failure, 88 ignored (71 assumptions) [seed: 3423860D0A8E2EF6] Total time: 60 minutes 27 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Recording test results Email was triggered for: Failure - Any Sending email for trigger: Failure - Any
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org