pong! Can you describe what behavior you're seeing and what you expect to see?
On Mon, Jul 17, 2017 at 1:30 AM, [email protected] <[email protected]> wrote: > ping! > > > > On 14/07/17 13:51, [email protected] wrote: > >> >> Hi >> >> One of our ensemble nodes is showing an erratic behavior: >> >> Jul 13 20:16:00 host12 zookeeper[26293]: [2017-07-13 20:16:00,500] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:16:00 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend( >> QuorumCnxManager.java:421) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.process(FastLeaderElection.java:486) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.run(FastLeaderElection.java:465) >> Jul 13 20:16:00 host12 zookeeper[26293]: #011at >> java.lang.Thread.run(Thread.java:748) >> Jul 13 20:17:00 host12 zookeeper[26293]: [2017-07-13 20:17:00,513] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:17:00 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend( >> QuorumCnxManager.java:421) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.process(FastLeaderElection.java:486) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.run(FastLeaderElection.java:465) >> Jul 13 20:17:00 host12 zookeeper[26293]: #011at >> java.lang.Thread.run(Thread.java:748) >> Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,649] >> WARN Connection broken for id 12, my id = 11, error = >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:17:26 host12 zookeeper[26293]: java.net.SocketException: >> Connection timed out (Read failed) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.net.SocketInputStream.socketRead0(Native Method) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.net.SocketInputStream.socketRead(SocketInputStream.java:116) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.net.SocketInputStream.read(SocketInputStream.java:171) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.net.SocketInputStream.read(SocketInputStream.java:141) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.net.SocketInputStream.read(SocketInputStream.java:224) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.io.DataInputStream.readInt(DataInputStream.java:387) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWork >> er.run(QuorumCnxManager.java:904) >> Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,649] >> WARN Interrupting SendWorker >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,650] >> WARN Interrupted while waiting for message on queue >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:17:26 host12 zookeeper[26293]: >> java.lang.InterruptedException >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$Condit >> ionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$Condit >> ionObject.awaitNanos(AbstractQueuedSynchronizer.java:2088) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQu >> eue.java:418) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.pollSend >> Queue(QuorumCnxManager.java:986) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.access$ >> 500(QuorumCnxManager.java:65) >> Jul 13 20:17:26 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWork >> er.run(QuorumCnxManager.java:824) >> Jul 13 20:17:26 host12 zookeeper[26293]: [2017-07-13 20:17:26,654] >> WARN Send worker leaving thread id 12 my id = 11 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:18:00 host12 zookeeper[26293]: [2017-07-13 20:18:00,501] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:18:00 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend( >> QuorumCnxManager.java:421) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.process(FastLeaderElection.java:486) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.run(FastLeaderElection.java:465) >> Jul 13 20:18:00 host12 zookeeper[26293]: #011at >> java.lang.Thread.run(Thread.java:748) >> Jul 13 20:18:05 host12 zookeeper[26293]: [2017-07-13 20:18:05,540] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:18:05 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend( >> QuorumCnxManager.java:421) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.process(FastLeaderElection.java:486) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection$Messen >> ger$WorkerSender.run(FastLeaderElection.java:465) >> Jul 13 20:18:05 host12 zookeeper[26293]: #011at >> java.lang.Thread.run(Thread.java:748) >> Jul 13 20:19:10 host12 zookeeper[26293]: [2017-07-13 20:19:10,548] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:19:10 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:19:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:19:15 host12 zookeeper[26293]: [2017-07-13 20:19:15,554] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:19:15 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:19:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:20:20 host12 zookeeper[26293]: [2017-07-13 20:20:20,561] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:20:20 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:20:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:20:25 host12 zookeeper[26293]: [2017-07-13 20:20:25,565] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:20:25 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:20:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:21:30 host12 zookeeper[26293]: [2017-07-13 20:21:30,569] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:21:30 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:21:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:21:35 host12 zookeeper[26293]: [2017-07-13 20:21:35,572] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:21:35 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:21:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:22:40 host12 zookeeper[26293]: [2017-07-13 20:22:40,575] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:22:40 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:22:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:22:45 host12 zookeeper[26293]: [2017-07-13 20:22:45,584] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:22:45 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:22:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:23:50 host12 zookeeper[26293]: [2017-07-13 20:23:50,591] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:23:50 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:23:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:23:55 host12 zookeeper[26293]: [2017-07-13 20:23:55,596] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:23:55 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:23:55 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:25:00 host12 zookeeper[26293]: [2017-07-13 20:25:00,600] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:25:00 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:25:00 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:25:05 host12 zookeeper[26293]: [2017-07-13 20:25:05,604] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:25:05 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:25:05 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:26:10 host12 zookeeper[26293]: [2017-07-13 20:26:10,612] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:26:10 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:26:10 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:26:15 host12 zookeeper[26293]: [2017-07-13 20:26:15,625] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:26:15 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:26:15 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:27:20 host12 zookeeper[26293]: [2017-07-13 20:27:20,632] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:27:20 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:27:20 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:27:25 host12 zookeeper[26293]: [2017-07-13 20:27:25,635] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:27:25 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:27:25 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:28:30 host12 zookeeper[26293]: [2017-07-13 20:28:30,640] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:28:30 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:28:30 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:28:35 host12 zookeeper[26293]: [2017-07-13 20:28:35,645] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:28:35 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:28:35 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:29:40 host12 zookeeper[26293]: [2017-07-13 20:29:40,649] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:29:40 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:29:40 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:29:45 host12 zookeeper[26293]: [2017-07-13 20:29:45,652] >> WARN Cannot open channel to 13 at election address /10.0.0.13:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:29:45 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:29:45 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:30:50 host12 zookeeper[26293]: [2017-07-13 20:30:50,657] >> WARN Cannot open channel to 12 at election address /10.0.0.12:3888 >> (org.apache.zookeeper.server.quorum.QuorumCnxManager) >> Jul 13 20:30:50 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:443) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectO >> ne(QuorumCnxManager.java:486) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumCnxManager.connectA >> ll(QuorumCnxManager.java:514) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.FastLeaderElection.lookFo >> rLeader(FastLeaderElection.java:919) >> Jul 13 20:30:50 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1109) >> Jul 13 20:30:54 host12 zookeeper[26293]: [2017-07-13 20:30:54,880] >> WARN Unexpected exception, tries=0, remaining init limit=6000, >> connecting to /10.0.0.12:2888 >> (org.apache.zookeeper.server.quorum.Learner) >> Jul 13 20:30:54 host12 zookeeper[26293]: >> java.net.SocketTimeoutException: connect timed out >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.PlainSocketImpl.socketConnect(Native Method) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSock >> etImpl.java:350) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPl >> ainSocketImpl.java:206) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocket >> Impl.java:188) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> java.net.Socket.connect(Socket.java:589) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.Learner.sockConnect(Learn >> er.java:227) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.Learner.connectToLeader( >> Learner.java:256) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.Follower.followLeader(Fol >> lower.java:76) >> Jul 13 20:30:54 host12 zookeeper[26293]: #011at >> org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer >> .java:1133) >> Jul 13 21:00:00 host12 zookeeper[26293]: [2017-07-13 21:00:00,522] >> WARN Got zxid 0x200007d99 expected 0x1 >> (org.apache.zookeeper.server.quorum.Learner) >> Jul 14 05:50:05 host12 zookeeper[26293]: [2017-07-14 05:50:05,509] >> WARN Unable to read additional data from client sessionid >> 0xc00bee319ed0001, likely client has closed socket >> (org.apache.zookeeper.server.NIOServerCnxn) >> >> As we are using docker, I was wondering if theres someone else having the >> same issues, or if there's a timeout setting (or anything else) we could >> set to improve our zookeeper ensemble stability. >> >> Thanks >> >> >> >
