Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.10-Linux/210/
Java: 32bit/jdk1.8.0_20 -client -XX:+UseConcMarkSweepGC (asserts: true)
1 tests failed.
FAILED: org.apache.solr.cloud.OverseerTest.testOverseerFailure
Error Message:
KeeperErrorCode = NoNode for
/collections/collection1/leader_elect/shard1/election
Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode
for /collections/collection1/leader_elect/shard1/election
at
__randomizedtesting.SeedInfo.seed([482CB93BE9A281DA:4C2436C8FB076EFB]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
at
org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:462)
at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:459)
at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:416)
at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:403)
at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:359)
at
org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:144)
at
org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:662)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at
org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
at
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
at
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at
com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11347 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> Creating dataDir:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build/solr-core/test/J0/./solr.cloud.OverseerTest-482CB93BE9A281DA-001/init-core-data-001
[junit4] 2> 388980 T953 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl
(false) and clientAuth (false)
[junit4] 2> 388981 T953 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> 388981 T953 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 388984 T953 oas.SolrTestCaseJ4.setUp ###Starting
testOverseerFailure
[junit4] 2> 388984 T953 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 388985 T954 oasc.ZkTestServer$ZKServerMain.runFromConfig
Starting server
[junit4] 2> 389085 T953 oasc.ZkTestServer.run start zk server on port:47310
[junit4] 2> 389086 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 389089 T960 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@ba4a91 name:ZooKeeperConnection
Watcher:127.0.0.1:47310 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 389089 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 389091 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 389093 T962 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1b15340 name:ZooKeeperConnection
Watcher:127.0.0.1:47310 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 389093 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 389093 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 389097 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 389099 T964 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@19a8514 name:ZooKeeperConnection
Watcher:127.0.0.1:47310/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 389099 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 389100 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 389102 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 389103 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 389104 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 389108 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 389109 T966 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1fee3ed name:ZooKeeperConnection
Watcher:127.0.0.1:47310/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 389109 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 389111 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 389113 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node1
[junit4] 2> 389124 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 389126 T970 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@154da01 name:ZooKeeperConnection
Watcher:127.0.0.1:47310/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 389126 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 389126 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 389131 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 389134 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 389135 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 389137 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:47310_solr
[junit4] 2> 389137 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 389139 T953 oasc.Overseer.start Overseer
(id=93089847943626756-127.0.0.1:47310_solr-n_0000000000) starting
[junit4] 2> 389140 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 389142 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 389147 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 389149 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 389151 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 389153 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 389157 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 389158 T971 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 389163 T972 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 390161 T974 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 390163 T971 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 390163 T971 oasc.Overseer$ClusterStateUpdater.createCollection
Create collection collection1 with shards [shard1]
[junit4] 2> 390164 T971 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 390166 T967 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 390166 T968 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 390662 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 390668 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard1
[junit4] 2> 390672 T974 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 390674 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 390676 T971 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 390676 T971 oasc.Overseer$ClusterStateUpdater.updateState
shard=shard1 is already registered
[junit4] 2> 390679 T974 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 390679 T968 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 390679 T967 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 391177 T972 oasc.OverseerCollectionProcessor.amILeader
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode =
Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:514)
[junit4] 2> at
org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:272)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 391178 T972 oasc.OverseerCollectionProcessor.amILeader
According to ZK I (id=93089847943626756-127.0.0.1:47310_solr-n_0000000000) am
no longer a leader.
[junit4] 2> 391733 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 391738 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 391740 T976 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@52bf38 name:ZooKeeperConnection
Watcher:127.0.0.1:47310/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 391740 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 391741 T953 oasc.Overseer.close Overseer
(id=93089847943626756-127.0.0.1:47310_solr-n_0000000000) closing
[junit4] 2> 391741 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 391741 T971 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:47310_solr
[junit4] 2> 391743 T977
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 391743 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 391746 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:47310_solr
[junit4] 2> 391746 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 391748 T953 oasc.Overseer.start Overseer
(id=93089847943626757-127.0.0.1:47310_solr-n_0000000001) starting
[junit4] 2> 391753 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 391754 T979 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 391754 T978 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 391756 T978 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 391757 T978 oasc.Overseer$ClusterStateUpdater.updateState
shard=shard1 is already registered
[junit4] 2> 391759 T981 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 391759 T967 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 391759 T968 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 391812 T981 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 391812 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 391819 T968 oasc.LeaderElector.checkIfIamLeader WARN Our node
is no longer in line to be leader
[junit4] 2> 391820 T968 oasc.LeaderElector$ElectionWatcher.process WARN
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode
for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:290)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:209)
[junit4] 2> at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at
java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 391830 T953 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:47310 47310
[junit4] 2> 392369 T954 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:47310 47310
[junit4] 2> 392370 T953 oas.SolrTestCaseJ4.tearDown ###Ending
testOverseerFailure
[junit4] 2> 392371 T953 oasc.Overseer.close Overseer
(id=93089847943626757-127.0.0.1:47310_solr-n_0000000001) closing
[junit4] 2> 392371 T978 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:47310_solr
[junit4] 2> 392373 T982
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerTest
-Dtests.method=testOverseerFailure -Dtests.seed=482CB93BE9A281DA
-Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_BH
-Dtests.timezone=Australia/Tasmania -Dtests.file.encoding=UTF-8
[junit4] ERROR 3.41s J0 | OverseerTest.testOverseerFailure <<<
[junit4] > Throwable #1:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode
for /collections/collection1/leader_elect/shard1/election
[junit4] > at
__randomizedtesting.SeedInfo.seed([482CB93BE9A281DA:4C2436C8FB076EFB]:0)
[junit4] > at
org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] > at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at
org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] > at
org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:462)
[junit4] > at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] > at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:459)
[junit4] > at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:416)
[junit4] > at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:403)
[junit4] > at
org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:359)
[junit4] > at
org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:144)
[junit4] > at
org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:662)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 392392 T953 oas.SolrTestCaseJ4.setUp ###Starting
testDoubleAssignment
[junit4] 2> 392393 T953 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 392394 T983 oasc.ZkTestServer$ZKServerMain.runFromConfig
Starting server
[junit4] 2> 392494 T953 oasc.ZkTestServer.run start zk server on port:42282
[junit4] 2> 392495 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 392497 T989 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@190d9cb name:ZooKeeperConnection
Watcher:127.0.0.1:42282/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 392498 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 392498 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 392500 T991 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@18eb2cc name:ZooKeeperConnection
Watcher:127.0.0.1:42282 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 392500 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 392502 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 392503 T993 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@b3ccfc name:ZooKeeperConnection
Watcher:127.0.0.1:42282 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 392503 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 392504 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 392507 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 392508 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 392510 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 392511 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 392513 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 392514 T995 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@4fa172 name:ZooKeeperConnection
Watcher:127.0.0.1:42282/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 392514 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 392515 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 392516 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node1
[junit4] 2> 392519 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 392520 T999 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1606c31 name:ZooKeeperConnection
Watcher:127.0.0.1:42282/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 392520 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 392520 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 392521 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 392523 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 392524 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 392525 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:42282_solr
[junit4] 2> 392525 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 392526 T953 oasc.Overseer.start Overseer
(id=93089848167104516-127.0.0.1:42282_solr-n_0000000000) starting
[junit4] 2> 392527 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 392530 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 392532 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 392535 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 392538 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 392541 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 392546 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 392547 T1001 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 392548 T1000 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 392550 T1000 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 392551 T1000
oasc.Overseer$ClusterStateUpdater.createCollection Create collection
collection1 with shards [shard1]
[junit4] 2> 392551 T1000 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 392554 T1003 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 392554 T997 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 392554 T996 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 393049 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 393055 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard1
[junit4] 2> 393059 T1003 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 393062 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393064 T1005 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1d0ba03 name:ZooKeeperConnection
Watcher:127.0.0.1:42282/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 393064 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393065 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 393067 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node1
[junit4] 2> 393070 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 393070 T1003 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 393071 T1000 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 393072 T1000 oasc.Overseer$ClusterStateUpdater.updateState
shard=shard1 is already registered
[junit4] 2> 393074 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard1
[junit4] 2> 393077 T1003 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 393180 T997 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 393180 T1006 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 393185 T953 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:42282 42282
[junit4] 2> 393742 T983 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:42282 42282
[junit4] 2> 393743 T953 oas.SolrTestCaseJ4.tearDown ###Ending
testDoubleAssignment
[junit4] 2> 393744 T953 oasc.Overseer.close Overseer
(id=93089848167104516-127.0.0.1:42282_solr-n_0000000000) closing
[junit4] 2> 393744 T1000 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:42282_solr
[junit4] 2> 393746 T1007
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 393749 T953 oas.SolrTestCaseJ4.setUp ###Starting
testBadQueueItem
[junit4] 2> 393750 T953 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 393750 T1008 oasc.ZkTestServer$ZKServerMain.runFromConfig
Starting server
[junit4] 2> 393850 T953 oasc.ZkTestServer.run start zk server on port:51014
[junit4] 2> 393851 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393854 T1014 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@11a4be9 name:ZooKeeperConnection
Watcher:127.0.0.1:51014 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 393855 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393857 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393858 T1016 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@b33449 name:ZooKeeperConnection
Watcher:127.0.0.1:51014 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 393859 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393859 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 393864 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393866 T1018 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@50c6ca name:ZooKeeperConnection
Watcher:127.0.0.1:51014/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 393866 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393867 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 393869 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393878 T1020 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@9c6f61 name:ZooKeeperConnection
Watcher:127.0.0.1:51014/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 393879 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393880 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 393881 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 393882 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 393884 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 393885 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:51014_solr
[junit4] 2> 393886 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 393887 T953 oasc.Overseer.start Overseer
(id=93089848255905795-127.0.0.1:51014_solr-n_0000000000) starting
[junit4] 2> 393888 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 393890 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 393891 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 393893 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 393895 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 393897 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 393899 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 393900 T1022 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 393900 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 393900 T1021 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 393902 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 393903 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 393905 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 393906 T1025 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@15c3b18 name:ZooKeeperConnection
Watcher:127.0.0.1:51014/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 393906 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 393907 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 393908 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/127.0.0.1
[junit4] 2> 393911 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 393912 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 393912 T1021
oasc.Overseer$ClusterStateUpdater.createCollection Create collection
collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 393912 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard2
[junit4] 2> 393914 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 393914 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 394411 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard2/election
[junit4] 2> 394417 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard2
[junit4] 2> 394420 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 394424 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 394425 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 394426 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard3
[junit4] 2> 394427 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 394428 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 394428 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 394922 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard3/election
[junit4] 2> 394926 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard3
[junit4] 2> 394929 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 394932 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 394933 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 394934 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 394935 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 394936 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 394937 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 395430 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 395435 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard1
[junit4] 2> 395439 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 395543 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 395543 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 395607 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 395609 T1021
oasc.Overseer$ClusterStateUpdater.checkKeyExistence ERROR Skipping invalid
Overseer message because it has no collection specified: {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core0",
[junit4] 2> "core_node_name":"node0",
[junit4] 2> "collection":"",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 395611 T1021
oasc.Overseer$ClusterStateUpdater.checkKeyExistence ERROR Skipping invalid
Overseer message because it has no collection specified: {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core0",
[junit4] 2> "core_node_name":"node0",
[junit4] 2> "collection":""}
[junit4] 2> 395613 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 395614 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 395614 T1021
oasc.Overseer$ClusterStateUpdater.createCollection Create collection
collection2 with shards [shard1, shard2, shard3]
[junit4] 2> 395614 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard2
[junit4] 2> 395718 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 395718 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 396100 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leader_elect/shard2/election
[junit4] 2> 396105 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leaders/shard2
[junit4] 2> 396108 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 396111 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 396111 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 396111 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard3
[junit4] 2> 396113 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 396113 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 396114 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 396609 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leader_elect/shard3/election
[junit4] 2> 396613 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leaders/shard3
[junit4] 2> 396617 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 396620 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 396620 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 396621 T1021 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 396622 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 396623 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 396623 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 397119 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leader_elect/shard1/election
[junit4] 2> 397124 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection2/leaders/shard1
[junit4] 2> 397128 T1028 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 397232 T1026 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 397232 T1027 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 397283 T953 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:51014 51014
[junit4] 2> 397902 T1022 oasc.OverseerCollectionProcessor.amILeader
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode =
Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:514)
[junit4] 2> at
org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:272)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 397956 T1022 oasc.OverseerCollectionProcessor.amILeader
According to ZK I (id=93089848255905795-127.0.0.1:51014_solr-n_0000000000) am
no longer a leader.
[junit4] 2> 397957 T1008 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:51014 51014
[junit4] 2> 397958 T953 oas.SolrTestCaseJ4.tearDown ###Ending
testBadQueueItem
[junit4] 2> 397959 T953 oasc.Overseer.close Overseer
(id=93089848255905795-127.0.0.1:51014_solr-n_0000000000) closing
[junit4] 2> 397959 T1021 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:51014_solr
[junit4] 2> 397960 T1029
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 397964 T953 oas.SolrTestCaseJ4.setUp ###Starting
testShardAssignment
[junit4] 2> 397965 T953 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 397965 T1030 oasc.ZkTestServer$ZKServerMain.runFromConfig
Starting server
[junit4] 2> 398065 T953 oasc.ZkTestServer.run start zk server on port:55712
[junit4] 2> 398066 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 398069 T1036 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@6e5406 name:ZooKeeperConnection
Watcher:127.0.0.1:55712 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 398069 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 398071 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 398074 T1038 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@d6a217 name:ZooKeeperConnection
Watcher:127.0.0.1:55712 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 398074 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 398075 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 398078 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 398079 T1040 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1463f8b name:ZooKeeperConnection
Watcher:127.0.0.1:55712/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 398080 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 398080 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 398081 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 398083 T1042 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1325300 name:ZooKeeperConnection
Watcher:127.0.0.1:55712/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 398083 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 398083 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 398084 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 398086 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 398087 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 398089 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:55712_solr
[junit4] 2> 398089 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 398090 T953 oasc.Overseer.start Overseer
(id=93089848532140035-127.0.0.1:55712_solr-n_0000000000) starting
[junit4] 2> 398092 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 398094 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 398096 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 398098 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 398102 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 398105 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 398108 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 398109 T1044 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 398109 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 398110 T1043 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 398112 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 398114 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 398117 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 398118 T1047 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1bcf404 name:ZooKeeperConnection
Watcher:127.0.0.1:55712/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 398118 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 398119 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 398121 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/127.0.0.1
[junit4] 2> 398127 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 398128 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 398129 T1043
oasc.Overseer$ClusterStateUpdater.createCollection Create collection
collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 398129 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard2
[junit4] 2> 398131 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 398131 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 398624 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard2/election
[junit4] 2> 398633 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard2
[junit4] 2> 398636 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 398639 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 398640 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 398640 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard3
[junit4] 2> 398641 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 398642 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 398643 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 399137 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard3/election
[junit4] 2> 399144 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard3
[junit4] 2> 399147 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 399150 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 399151 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 399151 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 399152 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 399153 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 399153 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 399648 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 399653 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leaders/shard1
[junit4] 2> 399656 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 399660 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 399660 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 399660 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard2
[junit4] 2> 399662 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 399662 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 399662 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 400158 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard2/election
[junit4] 2> 400164 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 400166 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 400166 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 400166 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard3
[junit4] 2> 400269 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 400269 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 400664 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard3/election
[junit4] 2> 400672 T1050 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 400674 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 400674 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Collection already exists with numShards=3
[junit4] 2> 400674 T1043 oasc.Overseer$ClusterStateUpdater.updateState
Assigning new node to shard shard=shard1
[junit4] 2> 400778 T1048 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 400778 T1049 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 401172 T953 oascc.SolrZkClient.makePath makePath:
/collections/collection1/leader_elect/shard1/election
[junit4] 2> 401181 T1051 oasc.LeaderElector$ElectionWatcher.process WARN
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode =
Session expired for /collections/collection1/leader_elect/shard2/election
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:290)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:209)
[junit4] 2> at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at
java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 401181 T1047 oaz.ClientCnxn$EventThread.processEvent ERROR
Error while calling watcher java.util.concurrent.RejectedExecutionException:
Task java.util.concurrent.FutureTask@c5d86a rejected from
java.util.concurrent.ThreadPoolExecutor@6294aa[Shutting down, pool size = 2,
active threads = 2, queued tasks = 0, completed tasks = 8]
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
[junit4] 2> at
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:206)
[junit4] 2> at
org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
[junit4] 2> at
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[junit4] 2>
[junit4] 2> 402680 T1048 oasc.LeaderElector$ElectionWatcher.process WARN
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode =
Session expired for /collections/collection1/leader_elect/shard3/election
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:290)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:287)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at
org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:209)
[junit4] 2> at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at
java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 402684 T953 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:55712 55712
[junit4] 2> 403256 T1030 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:55712 55712
[junit4] 2> 403258 T953 oas.SolrTestCaseJ4.tearDown ###Ending
testShardAssignment
[junit4] 2> 403258 T953 oasc.Overseer.close Overseer
(id=93089848532140035-127.0.0.1:55712_solr-n_0000000000) closing
[junit4] 2> 403259 T1043 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:55712_solr
[junit4] 2> 403260 T1053
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 403266 T953 oas.SolrTestCaseJ4.setUp ###Starting
testShardAssignmentBigger
[junit4] 2> 403267 T953 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 403268 T1054 oasc.ZkTestServer$ZKServerMain.runFromConfig
Starting server
[junit4] 2> 403368 T953 oasc.ZkTestServer.run start zk server on port:46977
[junit4] 2> 403369 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403371 T1060 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@33fed6 name:ZooKeeperConnection
Watcher:127.0.0.1:46977 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 403371 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403374 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403375 T1062 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1601c83 name:ZooKeeperConnection
Watcher:127.0.0.1:46977 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 403376 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403376 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 403379 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403381 T1064 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1b0d07f name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403381 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403382 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 403383 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403385 T1066 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@93586a name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403385 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403385 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 403387 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 403388 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 403389 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 403391 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:46977_solr
[junit4] 2> 403391 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 403393 T953 oasc.Overseer.start Overseer
(id=93089848879677443-127.0.0.1:46977_solr-n_0000000000) starting
[junit4] 2> 403394 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 403396 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 403399 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 403400 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 403402 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 403404 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 403407 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 403407 T1068 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 403408 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 403408 T1067 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 403409 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 403410 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403412 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403413 T1071 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@111c320 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403414 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403415 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403416 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node0
[junit4] 2> 403419 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403420 T1075 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@b45b85 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403420 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403421 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403423 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node1
[junit4] 2> 403425 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403426 T1078 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1ec5cc4 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403426 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403427 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403429 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node2
[junit4] 2> 403446 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403447 T1081 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@1a015db name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403448 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403449 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403453 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node3
[junit4] 2> 403460 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403476 T1084 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@e7cccc name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403476 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403477 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403479 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node4
[junit4] 2> 403500 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403500 T1086 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@cdcc26 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403501 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403502 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403504 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node5
[junit4] 2> 403513 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403514 T1090 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@12ea9c7 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403514 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403515 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403517 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node6
[junit4] 2> 403520 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403522 T1093 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@b764b4 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403522 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403523 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403526 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node7
[junit4] 2> 403530 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403532 T1096 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@d05542 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403532 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403533 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403535 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node8
[junit4] 2> 403545 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403546 T1099 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@8805c2 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403547 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403548 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403550 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node9
[junit4] 2> 403554 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403555 T1102 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@eeea45 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403555 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403556 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403558 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node10
[junit4] 2> 403564 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 403566 T1105 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@123a228 name:ZooKeeperConnection
Watcher:127.0.0.1:46977/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 403566 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 403567 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 403569 T953 oascc.SolrZkClient.makePath makePath:
/live_nodes/node11
[junit4] 2> 403574 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[j
[...truncated too long message...]
nected to ZooKeeper
[junit4] 2> 457399 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 457400 T1607 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@fe04da name:ZooKeeperConnection
Watcher:127.0.0.1:38493 got event WatchedEvent state:SyncConnected type:None
path:null path:null type:None
[junit4] 2> 457400 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 457401 T953 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 457403 T953 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 457405 T953 oascc.SolrZkClient.makePath makePath:
/clusterstate.json
[junit4] 2> 457407 T953 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 457408 T953
oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state
from ZooKeeper...
[junit4] 2> 457410 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue-work
[junit4] 2> 457425 T953 oascc.ConnectionManager.waitForConnected Waiting
for client to connect to ZooKeeper
[junit4] 2> 457426 T1609 oascc.ConnectionManager.process Watcher
org.apache.solr.common.cloud.ConnectionManager@10a77c3 name:ZooKeeperConnection
Watcher:127.0.0.1:38493/solr got event WatchedEvent state:SyncConnected
type:None path:null path:null type:None
[junit4] 2> 457427 T953 oascc.ConnectionManager.waitForConnected Client is
connected to ZooKeeper
[junit4] 2> 457427 T953 oasu.UpdateShardHandler.<init> Creating
UpdateShardHandler HTTP client with params: retry=false
[junit4] 2> 457428 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect
[junit4] 2> 457429 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/election
[junit4] 2> 457431 T953 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 457432 T953 oasc.OverseerElectionContext.runLeaderProcess I am
going to be the leader 127.0.0.1:38493_solr
[junit4] 2> 457433 T953 oascc.SolrZkClient.makePath makePath:
/overseer_elect/leader
[junit4] 2> 457434 T953 oasc.Overseer.start Overseer
(id=93089852420259843-127.0.0.1:38493_solr-n_0000000000) starting
[junit4] 2> 457435 T953 oascc.SolrZkClient.makePath makePath:
/overseer/queue
[junit4] 2> 457437 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-failure
[junit4] 2> 457439 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-running
[junit4] 2> 457440 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-map-completed
[junit4] 2> 457442 T953 oascc.SolrZkClient.makePath makePath:
/overseer/collection-queue-work
[junit4] 2> 457445 T953 oasc.OverseerAutoReplicaFailoverThread.<init>
Starting OverseerAutoReplicaFailoverThread
autoReplicaFailoverWorkLoopDelay=10000
autoReplicaFailoverWaitAfterExpiration=30000
autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 457445 T1611 oasc.OverseerCollectionProcessor.run Process
current queue of collection creations
[junit4] 2> 457447 T1610 oasc.Overseer$ClusterStateUpdater.run Replaying
operations from work queue.
[junit4] 2> 457447 T1610 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr";,
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 457448 T1613 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 457450 T1610 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr";,
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 457450 T1613 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 457451 T1610 oasc.Overseer$ClusterStateUpdater.run Starting to
work on the main queue
[junit4] 2> 457452 T1610 oasc.Overseer$ClusterStateUpdater.updateState
Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr";,
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 457454 T1614 oasc.DistributedQueue$LatchChildWatcher.process
LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type
NodeChildrenChanged
[junit4] 2> 457454 T1613 oascc.ZkStateReader$2.process A cluster state
change: WatchedEvent state:SyncConnected type:NodeDataChanged
path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 457549 T953 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:38493 38493
[junit4] 2> 457962 T1597 oasc.ZkTestServer.send4LetterWord connecting to
127.0.0.1:38493 38493
[junit4] 2> 457963 T953 oas.SolrTestCaseJ4.tearDown ###Ending testReplay
[junit4] 2> 457964 T953 oasc.Overseer.close Overseer
(id=93089852420259843-127.0.0.1:38493_solr-n_0000000000) closing
[junit4] 2> 457964 T1610 oasc.Overseer$ClusterStateUpdater.run Overseer
Loop exiting : 127.0.0.1:38493_solr
[junit4] 2> 457965 T1615
oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read
the data org.apache.zookeeper.KeeperException$SessionExpiredException:
KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at
org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at
org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at
org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 460966 T953 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build/solr-core/test/J0/./solr.cloud.OverseerTest-482CB93BE9A281DA-001
[junit4] 2> NOTE: test params are:
codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY,
chunkSize=355),
termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY,
chunkSize=355)), sim=DefaultSimilarity, locale=ar_BH,
timezone=Australia/Tasmania
[junit4] 2> NOTE: Linux 3.13.0-43-generic i386/Oracle Corporation 1.8.0_20
(32-bit)/cpus=8,threads=1,free=82580360,total=140730368
[junit4] 2> NOTE: All tests run in this JVM:
[DistribDocExpirationUpdateProcessorTest, XmlUpdateRequestHandlerTest,
TestDynamicFieldResource, TestBM25SimilarityFactory, TestHashPartitioner,
PathHierarchyTokenizerFactoryTest, DistributedSuggestComponentTest,
MigrateRouteKeyTest, TestDocSet, TestFieldTypeResource,
TestReloadAndDeleteDocs, AnalyticsMergeStrategyTest, TestNonNRTOpen,
TestStressVersions, ZkNodePropsTest, TestQuerySenderListener,
TestStressReorder, HdfsWriteToMultipleCollectionsTest, TestPivotHelperCode,
TestSolrJ, TestTrie, TestLuceneMatchVersion,
DeleteLastCustomShardedReplicaTest, TestSolrDeletionPolicy1,
TestSolrDeletionPolicy2, OverseerTest]
[junit4] Completed on J0 in 72.01s, 9 tests, 1 error <<< FAILURES!
[...truncated 1162 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:474: The following
error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:454: The following
error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:45: The following
error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/extra-targets.xml:37: The
following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build.xml:189: The
following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/common-build.xml:494:
The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:1351:
The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:966:
There were test failures: 421 suites, 1757 tests, 1 error, 26 ignored (19
assumptions)
Total time: 114 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.8.0_20 -client
-XX:+UseConcMarkSweepGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org
