[
https://issues.apache.org/jira/browse/IGNITE-11983?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dmitry Lazurkin updated IGNITE-11983:
-------------------------------------
Description:
Node *ignite-2*/*ignite-3* has inifite logs about
TcpDiscoveryStatusCheckMessage with id
832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb.
h2. Cluster topology
*ignite-1 (f170bfc0-3dec-4b7b-8815-99ca641abc2f)*
{noformat}
2019-07-15 12:13:16,076 [RMI TCP Connection(10)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
order=2, intOrder=2, lastExchangeTime=1563192795758, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1562343474727, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1563192795758, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562343561117, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562343652607, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562851741145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1562853582125, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=12,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
*ignite-2 (d47598e1-bbfd-4789-ba25-b1af6dc147bf, coordinator)*
{noformat}
2019-07-15 12:11:29,679 [RMI TCP Connection(88)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
order=1, intOrder=1, lastExchangeTime=1563192689675, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1563192689675, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1562343474657, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562343561102, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562343652597, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562851741133, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1562853582098, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=12,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
*ignite-3 (6a182cef-25dc-4ee6-8307-a62ec164f466, id before restart -
f507bad3-ea78-40a6-81ce-ab415ab871bb)*
{noformat}
2019-07-15 12:06:33,828 [RMI TCP Connection(39)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=18, intOrder=12, lastExchangeTime=1563192393823, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1562853582198, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1562853582208, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562853582178, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562853582188, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562853582198, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1563192393823, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=0,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
h2. Logs (inifite repeated)
Please, don't check time, logs from different moments.
*ignite-2 (coordinator)*
{noformat}
2019-07-15 12:16:29,722 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,724 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,725 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,726 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,728 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,729 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,730 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,733 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,734 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
{noformat}
*ignite-3*
{noformat}
2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery accepted incoming connection [rmtAddr=/10.48.44.2,
rmtPort=57737]
2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery spawning a new thread for connection [rmtAddr=/10.48.44.2,
rmtPort=57737]
2019-07-15 12:47:15,145 [tcp-disco-sock-reader-#79198414] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
[rmtAddr=/10.48.44.2:57737, rmtPort=57737]
2019-07-15 12:47:15,146 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
[nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] TRACE
o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
[nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] ERROR
o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
[sock=Socket[addr=/10.48.44.2,port=57737,localport=47500],
locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
given class loader:
org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
at
org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
Caused by: java.io.EOFException: null
at
java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
at
java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
at
java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
at
org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
... 4 common frames omitted
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
[rmtAddr=/10.48.44.2:57737, rmtPort=57737
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
tcp-disco-sock-reader-#79198414
2019-07-15 12:47:15,148 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
[msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery accepted incoming connection [rmtAddr=/10.48.44.2,
rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery spawning a new thread for connection [rmtAddr=/10.48.44.2,
rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
[rmtAddr=/10.48.44.2:59669, rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
[nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
2019-07-15 12:47:15,150 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] TRACE
o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
[nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] ERROR
o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
[sock=Socket[addr=/10.48.44.2,port=59669,localport=47500],
locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
given class loader:
org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
at
org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
Caused by: java.io.EOFException: null
at
java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
at
java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
at
java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
at
org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
... 4 common frames omitted
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
[rmtAddr=/10.48.44.2:59669, rmtPort=59669
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
tcp-disco-sock-reader-#79198415
2019-07-15 12:47:15,152 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
[msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
{noformat}
was:
Node *ignite-2*/*ignite-3* has inifite logs about
TcpDiscoveryStatusCheckMessage with id
832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb.
h2. Cluster topology
*ignite-1 (f170bfc0-3dec-4b7b-8815-99ca641abc2f)*
{noformat}
2019-07-15 12:13:16,076 [RMI TCP Connection(10)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
order=2, intOrder=2, lastExchangeTime=1563192795758, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1562343474727, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1563192795758, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562343561117, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562343652607, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562851741145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1562853582125, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=12,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
*ignite-2 (d47598e1-bbfd-4789-ba25-b1af6dc147bf, coordinator)*
{noformat}
2019-07-15 12:11:29,679 [RMI TCP Connection(88)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
order=1, intOrder=1, lastExchangeTime=1563192689675, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1563192689675, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1562343474657, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562343561102, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562343652597, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562851741133, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1562853582098, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=12,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
*ignite-3 (6a182cef-25dc-4ee6-8307-a62ec164f466, id before restart -
f507bad3-ea78-40a6-81ce-ab415ab871bb)*
{noformat}
2019-07-15 12:06:33,828 [RMI TCP Connection(39)-127.0.0.1] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
locNode=TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=18, intOrder=12, lastExchangeTime=1563192393823, loc=true,
ver=2.7.0#20181130-sha1:256ae401, isClient=false],
nodes=[
TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf, addrs=[10.48.44.2],
sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500, order=1, intOrder=1,
lastExchangeTime=1562853582198, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f, addrs=[10.48.44.1],
sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500, order=2, intOrder=2,
lastExchangeTime=1562853582208, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false],
TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
intOrder=6, lastExchangeTime=1562853582178, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=true],
TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645, addrs=[10.44.151.8],
sockAddrs=[/10.44.151.8:0], discPort=0, order=9, intOrder=7,
lastExchangeTime=1562853582188, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7, addrs=[10.37.26.56],
sockAddrs=[/10.37.26.56:0], discPort=0, order=14, intOrder=10,
lastExchangeTime=1562853582198, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=true],
TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=18, intOrder=12,
lastExchangeTime=1563192393823, loc=true, ver=2.7.0#20181130-sha1:256ae401,
isClient=false]], topVer=18, nodeOrder=12, maxInternalOrder=0,
minNodeVer=2.7.0#20181130-sha1:256ae401]
{noformat}
h2. Logs (inifite repeated)
*ignite-2 (coordinator)*
{noformat}
2019-07-15 12:16:29,722 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,724 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,725 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,726 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,728 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,729 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,730 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Message has been sent directly to address [msg=TcpDiscoveryStatusCheckMessage
[creatorNode=TcpDiscoveryNode [id=f507bad3-ea78-40a6-81ce-ab415ab871bb,
addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
order=16, intOrder=11, lastExchangeTime=1563192989720, loc=false,
ver=2.7.0#20181130-sha1:256ae401, isClient=false], failedNodeId=null, status=2,
super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
addr=ignite-3/10.48.44.3:47500, rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG o.a.i.s.d.tcp.TcpDiscoverySpi
- Responded to status check message
[recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
2019-07-15 12:16:29,733 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:16:29,734 [tcp-disco-sock-reader-#22] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
{noformat}
*ignite-3*
{noformat}
2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery accepted incoming connection [rmtAddr=/10.48.44.2,
rmtPort=57737]
2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery spawning a new thread for connection [rmtAddr=/10.48.44.2,
rmtPort=57737]
2019-07-15 12:47:15,145 [tcp-disco-sock-reader-#79198414] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
[rmtAddr=/10.48.44.2:57737, rmtPort=57737]
2019-07-15 12:47:15,146 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
[nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] TRACE
o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
[nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] ERROR
o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
[sock=Socket[addr=/10.48.44.2,port=57737,localport=47500],
locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
given class loader:
org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
at
org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
Caused by: java.io.EOFException: null
at
java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
at
java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
at
java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
at
org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
... 4 common frames omitted
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
[rmtAddr=/10.48.44.2:57737, rmtPort=57737
2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
tcp-disco-sock-reader-#79198414
2019-07-15 12:47:15,148 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
[msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery accepted incoming connection [rmtAddr=/10.48.44.2,
rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO o.a.i.s.d.tcp.TcpDiscoverySpi
- TCP discovery spawning a new thread for connection [rmtAddr=/10.48.44.2,
rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
[rmtAddr=/10.48.44.2:59669, rmtPort=59669]
2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
[nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
2019-07-15 12:47:15,150 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
[cls=TcpDiscoveryStatusCheckMessage,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] TRACE
o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
[nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] ERROR
o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
[sock=Socket[addr=/10.48.44.2,port=59669,localport=47500],
locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
given class loader:
org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
at
org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
Caused by: java.io.EOFException: null
at
java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
at
java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
at
java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
at
org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
at
org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
... 4 common frames omitted
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] INFO
o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
[rmtAddr=/10.48.44.2:59669, rmtPort=59669
2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
tcp-disco-sock-reader-#79198415
2019-07-15 12:47:15,152 [tcp-disco-msg-worker-#2] DEBUG
o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
[msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
[id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
isClient=false], failedNodeId=null, status=2, super=TcpDiscoveryAbstractMessage
[sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
{noformat}
> TcpDiscoverySpi: Infinite circulation of TcpDiscoveryStatusCheckMessage after
> node restart
> ------------------------------------------------------------------------------------------
>
> Key: IGNITE-11983
> URL: https://issues.apache.org/jira/browse/IGNITE-11983
> Project: Ignite
> Issue Type: Bug
> Components: general
> Affects Versions: 2.7
> Reporter: Dmitry Lazurkin
> Priority: Major
>
> Node *ignite-2*/*ignite-3* has inifite logs about
> TcpDiscoveryStatusCheckMessage with id
> 832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb.
> h2. Cluster topology
> *ignite-1 (f170bfc0-3dec-4b7b-8815-99ca641abc2f)*
> {noformat}
> 2019-07-15 12:13:16,076 [RMI TCP Connection(10)-127.0.0.1] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
> locNode=TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
> addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
> order=2, intOrder=2, lastExchangeTime=1563192795758, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> nodes=[
> TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
> order=1, intOrder=1, lastExchangeTime=1562343474727, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
> addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
> order=2, intOrder=2, lastExchangeTime=1563192795758, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
> addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
> intOrder=6, lastExchangeTime=1562343561117, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645,
> addrs=[10.44.151.8], sockAddrs=[/10.44.151.8:0], discPort=0, order=9,
> intOrder=7, lastExchangeTime=1562343652607, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7,
> addrs=[10.37.26.56], sockAddrs=[/10.37.26.56:0], discPort=0, order=14,
> intOrder=10, lastExchangeTime=1562851741145, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
> addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
> order=18, intOrder=12, lastExchangeTime=1562853582125, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false]], topVer=18, nodeOrder=12,
> maxInternalOrder=12, minNodeVer=2.7.0#20181130-sha1:256ae401]
> {noformat}
> *ignite-2 (d47598e1-bbfd-4789-ba25-b1af6dc147bf, coordinator)*
> {noformat}
> 2019-07-15 12:11:29,679 [RMI TCP Connection(88)-127.0.0.1] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
> locNode=TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
> order=1, intOrder=1, lastExchangeTime=1563192689675, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> nodes=[
> TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
> order=1, intOrder=1, lastExchangeTime=1563192689675, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
> addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
> order=2, intOrder=2, lastExchangeTime=1562343474657, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
> addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
> intOrder=6, lastExchangeTime=1562343561102, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645,
> addrs=[10.44.151.8], sockAddrs=[/10.44.151.8:0], discPort=0, order=9,
> intOrder=7, lastExchangeTime=1562343652597, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7,
> addrs=[10.37.26.56], sockAddrs=[/10.37.26.56:0], discPort=0, order=14,
> intOrder=10, lastExchangeTime=1562851741133, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
> addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
> order=18, intOrder=12, lastExchangeTime=1562853582098, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false]], topVer=18, nodeOrder=12,
> maxInternalOrder=12, minNodeVer=2.7.0#20181130-sha1:256ae401]
> {noformat}
> *ignite-3 (6a182cef-25dc-4ee6-8307-a62ec164f466, id before restart -
> f507bad3-ea78-40a6-81ce-ab415ab871bb)*
> {noformat}
> 2019-07-15 12:06:33,828 [RMI TCP Connection(39)-127.0.0.1] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TcpDiscoveryNodesRing [
> locNode=TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
> addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
> order=18, intOrder=12, lastExchangeTime=1563192393823, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> nodes=[
> TcpDiscoveryNode [id=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> addrs=[10.48.44.2], sockAddrs=[ignite-2/10.48.44.2:47500], discPort=47500,
> order=1, intOrder=1, lastExchangeTime=1562853582198, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=f170bfc0-3dec-4b7b-8815-99ca641abc2f,
> addrs=[10.48.44.1], sockAddrs=[ignite-1/10.48.44.1:47500], discPort=47500,
> order=2, intOrder=2, lastExchangeTime=1562853582208, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false],
> TcpDiscoveryNode [id=b400aa0a-1d9b-4165-96ae-d75aba2d1bb8,
> addrs=[10.40.105.88], sockAddrs=[/10.40.105.88:0], discPort=0, order=7,
> intOrder=6, lastExchangeTime=1562853582178, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=618b2910-071d-4247-86f3-1d40237e7645,
> addrs=[10.44.151.8], sockAddrs=[/10.44.151.8:0], discPort=0, order=9,
> intOrder=7, lastExchangeTime=1562853582188, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=2b033aed-132d-4341-b1cf-6f45a21619b7,
> addrs=[10.37.26.56], sockAddrs=[/10.37.26.56:0], discPort=0, order=14,
> intOrder=10, lastExchangeTime=1562853582198, loc=false,
> ver=2.7.0#20181130-sha1:256ae401, isClient=true],
> TcpDiscoveryNode [id=6a182cef-25dc-4ee6-8307-a62ec164f466,
> addrs=[10.48.44.3], sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500,
> order=18, intOrder=12, lastExchangeTime=1563192393823, loc=true,
> ver=2.7.0#20181130-sha1:256ae401, isClient=false]], topVer=18, nodeOrder=12,
> maxInternalOrder=0, minNodeVer=2.7.0#20181130-sha1:256ae401]
> {noformat}
> h2. Logs (inifite repeated)
> Please, don't check time, logs from different moments.
> *ignite-2 (coordinator)*
> {noformat}
> 2019-07-15 12:16:29,722 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
> [cls=TcpDiscoveryStatusCheckMessage,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
> 2019-07-15 12:16:29,724 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent directly to address
> [msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
> addr=ignite-3/10.48.44.3:47500,
> rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
> 2019-07-15 12:16:29,725 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Responded to status check message
> [recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
> 2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:16:29,726 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:16:29,726 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
> [cls=TcpDiscoveryStatusCheckMessage,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
> 2019-07-15 12:16:29,728 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent directly to address
> [msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
> addr=ignite-3/10.48.44.3:47500,
> rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
> 2019-07-15 12:16:29,729 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Responded to status check message
> [recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
> 2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:16:29,730 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:16:29,730 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
> [cls=TcpDiscoveryStatusCheckMessage,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
> 2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent directly to address
> [msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989720, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
> addr=ignite-3/10.48.44.3:47500,
> rmtNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466]
> 2019-07-15 12:16:29,732 [disco-pool-#51062] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Responded to status check message
> [recipient=f507bad3-ea78-40a6-81ce-ab415ab871bb, status=2]
> 2019-07-15 12:16:29,733 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:16:29,734 [tcp-disco-sock-reader-#22] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563192989730, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> {noformat}
> *ignite-3*
> {noformat}
> 2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TCP discovery accepted incoming connection
> [rmtAddr=/10.48.44.2, rmtPort=57737]
> 2019-07-15 12:47:15,145 [tcp-disco-srvr-#3] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TCP discovery spawning a new thread for
> connection [rmtAddr=/10.48.44.2, rmtPort=57737]
> 2019-07-15 12:47:15,145 [tcp-disco-sock-reader-#79198414] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
> [rmtAddr=/10.48.44.2:57737, rmtPort=57737]
> 2019-07-15 12:47:15,146 [tcp-disco-sock-reader-#79198414] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
> [nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
> 2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
> [cls=TcpDiscoveryStatusCheckMessage,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
> 2019-07-15 12:47:15,147 [tcp-disco-msg-worker-#2] TRACE
> o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
> [nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
> 2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] ERROR
> o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
> [sock=Socket[addr=/10.48.44.2,port=57737,localport=47500],
> locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
> org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
> given class loader:
> org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
> at
> org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
> at
> org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
> at
> org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
> at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
> Caused by: java.io.EOFException: null
> at
> java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
> at
> java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
> at
> java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
> at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
> at
> org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
> at
> org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
> ... 4 common frames omitted
> 2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
> [rmtAddr=/10.48.44.2:57737, rmtPort=57737
> 2019-07-15 12:47:15,147 [tcp-disco-sock-reader-#79198414] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
> tcp-disco-sock-reader-#79198414
> 2019-07-15 12:47:15,148 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
> [msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
> next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
> 2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TCP discovery accepted incoming connection
> [rmtAddr=/10.48.44.2, rmtPort=59669]
> 2019-07-15 12:47:15,149 [tcp-disco-srvr-#3] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - TCP discovery spawning a new thread for
> connection [rmtAddr=/10.48.44.2, rmtPort=59669]
> 2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - Started serving remote node connection
> [rmtAddr=/10.48.44.2:59669, rmtPort=59669]
> 2019-07-15 12:47:15,149 [tcp-disco-sock-reader-#79198415] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Initialized connection with remote node
> [nodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, client=false]
> 2019-07-15 12:47:15,150 [tcp-disco-sock-reader-#79198415] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been received:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been added to queue:
> TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]]
> 2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Processing message
> [cls=TcpDiscoveryStatusCheckMessage,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb]
> 2019-07-15 12:47:15,151 [tcp-disco-msg-worker-#2] TRACE
> o.a.i.s.d.tcp.TcpDiscoverySpi - Next node remains the same
> [nextId=d47598e1-bbfd-4789-ba25-b1af6dc147bf, nextOrder=1]
> 2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] ERROR
> o.a.i.s.d.tcp.TcpDiscoverySpi - Caught exception on message read
> [sock=Socket[addr=/10.48.44.2,port=59669,localport=47500],
> locNodeId=6a182cef-25dc-4ee6-8307-a62ec164f466,
> rmtNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf]
> org.apache.ignite.IgniteCheckedException: Failed to deserialize object with
> given class loader:
> org.springframework.boot.loader.LaunchedURLClassLoader@166fa74d
> at
> org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:147)
> at
> org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94)
> at
> org.apache.ignite.internal.util.IgniteUtils.unmarshal(IgniteUtils.java:10085)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:6246)
> at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62)
> Caused by: java.io.EOFException: null
> at
> java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2681)
> at
> java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156)
> at
> java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862)
> at java.io.ObjectInputStream.<init>(ObjectInputStream.java:358)
> at
> org.apache.ignite.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:43)
> at
> org.apache.ignite.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:137)
> ... 4 common frames omitted
> 2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] INFO
> o.a.i.s.d.tcp.TcpDiscoverySpi - Finished serving remote node connection
> [rmtAddr=/10.48.44.2:59669, rmtPort=59669
> 2019-07-15 12:47:15,151 [tcp-disco-sock-reader-#79198415] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Grid runnable finished normally:
> tcp-disco-sock-reader-#79198415
> 2019-07-15 12:47:15,152 [tcp-disco-msg-worker-#2] DEBUG
> o.a.i.s.d.tcp.TcpDiscoverySpi - Message has been sent to next node
> [msg=TcpDiscoveryStatusCheckMessage [creatorNode=TcpDiscoveryNode
> [id=f507bad3-ea78-40a6-81ce-ab415ab871bb, addrs=[10.48.44.3],
> sockAddrs=[ignite-3/10.48.44.3:47500], discPort=47500, order=16, intOrder=11,
> lastExchangeTime=1563194835145, loc=false, ver=2.7.0#20181130-sha1:256ae401,
> isClient=false], failedNodeId=null, status=2,
> super=TcpDiscoveryAbstractMessage
> [sndNodeId=d47598e1-bbfd-4789-ba25-b1af6dc147bf,
> id=832a931eb61-f507bad3-ea78-40a6-81ce-ab415ab871bb, verifierNodeId=null,
> topVer=0, pendingIdx=0, failedNodes=null, isClient=false]],
> next=d47598e1-bbfd-4789-ba25-b1af6dc147bf, res=1]
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)
