Hi, Any one to the rescue here please.I am finding it hard to understand these logs. When this starts happening, ignite grid just stops working. Nothing works unless grid is restarted.
Any helpful would be grateful. Error log: [2017-09-07/13:16:47.856/BST][WARN ][grid-timeout-worker-#15%null%][o.a.i.i.u.t.G] - >>> Possible starvation in striped pool: sys-stripe-3-#4%null% [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786161899, order=1504763782702, nodeOrder=27], writeVer=GridCacheVersion [topVer=116242068, time=1504786161899, order=1504763782701, nodeOrder=27], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[28]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=667176, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786161967, order=1504763685550, nodeOrder=34], writeVer=GridCacheVersion [topVer=116242068, time=1504786161967, order=1504763685549, nodeOrder=34], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[33]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=189850, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162325, order=1504763687249, nodeOrder=24], writeVer=GridCacheVersion [topVer=116242068, time=1504786162325, order=1504763687248, nodeOrder=24], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[31]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=182619, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162376, order=1504763679517, nodeOrder=28], writeVer=GridCacheVersion [topVer=116242068, time=1504786162376, order=1504763679516, nodeOrder=28], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[39]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=170147, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162315, order=1504763735961, nodeOrder=20], writeVer=GridCacheVersion [topVer=116242068, time=1504786162315, order=1504763735960, nodeOrder=20], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[38]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=585895, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162357, order=1504763735966, nodeOrder=20], writeVer=GridCacheVersion [topVer=116242068, time=1504786162357, order=1504763735965, nodeOrder=20], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[24]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=585920, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162382, order=1504763737150, nodeOrder=16], writeVer=GridCacheVersion [topVer=116242068, time=1504786162382, order=1504763737149, nodeOrder=16], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[20]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=573418, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridCacheMessage [msgId=141493, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162411, order=1504763782706, nodeOrder=27], writeVer=GridCacheVersion [topVer=116242068, time=1504786162411, order=1504763782705, nodeOrder=27], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[11]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=667233, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [futVer=GridCacheVersion [topVer=116242068, time=1504786162419, order=1504763679523, nodeOrder=28], writeVer=GridCacheVersion [topVer=116242068, time=1504786162419, order=1504763679522, nodeOrder=28], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], keys=[KeyCacheObjectImpl [val=null, hasValBytes=true]], vals=[CacheObjectImpl [val=null, hasValBytes=true]], prevVals=null, ttls=GridLongList [idx=1, arr=[1800000]], conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, syncMode=FULL_ASYNC, nearKeys=null, nearVals=null, forceTransformBackups=false, subjId=fbbc8445-c275-4459-9ca6-291918131ee1, taskNameHash=0, updateCntrs=GridLongList [idx=1, arr=[15]], keepBinary=false, flags=0, super=GridCacheMessage [msgId=170221, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridCacheMessage [msgId=192791, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridCacheMessage [msgId=573449, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=192879, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=558075, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=189946, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=189969, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], parent=GridCacheMessage [msgId=622388, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearSingleGetResponse [futId=1504763483968, res=CacheObjectImpl [val=null, hasValBytes=true], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], err=null, flags=0]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=158389, depInfo=null, err=null, skipPrepare=false, cacheId=99252427, cacheId=99252427]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearLockResponse [pending=[], miniId=77794ea5e51-0d31f05d-74bc-4dd3-9cbb-7bc1ce01eb7f, dhtVers=[GridCacheVersion [topVer=116242068, time=1504785950610, order=1504763684987, nodeOrder=30]], mappedVers=[GridCacheVersion [topVer=116242068, time=1504786200287, order=1504763685630, nodeOrder=36]], clientRemapVer=null, super=GridDistributedLockResponse [futId=67794ea5e51-0d31f05d-74bc-4dd3-9cbb-7bc1ce01eb7f, err=null, vals=[CacheObjectImpl [val=null, hasValBytes=true]], super=GridDistributedBaseMessage [ver=GridCacheVersion [topVer=116242068, time=1504786200287, order=1504763685630, nodeOrder=36], committedVers=[], rolledbackVers=[], cnt=0, super=GridCacheMessage [msgId=141623, depInfo=null, err=null, skipPrepare=false, cacheId=1756769137, cacheId=1756769137]]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [val=null, hasValBytes=true], val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null, super=GridCacheMessage [msgId=192961, depInfo=null, err=null, skipPrepare=false, cacheId=611538803, cacheId=611538803]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearSingleGetResponse [futId=1504763485104, res=CacheObjectImpl [val=null, hasValBytes=true], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], err=null, flags=0]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearSingleGetRequest [futId=1504763442204, key=KeyCacheObjectImpl [val=null, hasValBytes=true], partId=772, flags=1, topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], subjId=dd2ea872-000c-4754-bcc1-76d540044141, taskNameHash=0, accessTtl=-1]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearSingleGetResponse [futId=1504763487596, res=CacheObjectImpl [val=null, hasValBytes=true], topVer=AffinityTopologyVersion [topVer=36, minorTopVer=19], err=null, flags=0]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=CacheContinuousQueryBatchAck [routineId=be911e64-5195-4dea-aafc-886675f2f0de, updateCntrs={771=103, 5=103, 135=103, 9=103, 138=142, 524=103, 397=103, 782=103, 783=103, 912=131, 145=142, 657=103, 786=103, 531=103, 792=103, 155=140, 539=103, 284=103, 285=103, 157=141, 669=103, 799=103, 33=103, 801=103, 293=103, 937=127, 45=103, 945=130, 308=103, 565=103, 438=105, 183=141, 695=103, 697=103, 314=103, 315=103, 829=103, 957=103, 831=103, 832=103, 576=103, 193=138, 706=103, 963=106, 451=103, 69=136, 965=106, 713=103, 714=103, 77=103, 462=103, 338=103, 82=103, 211=142, 470=103, 985=103, 729=103, 731=103, 990=103, 863=103, 994=103, 229=142, 741=103, 745=103, 489=103, 618=103, 620=103, 1005=103, 749=103, 238=118, 366=103, 1011=103, 501=103, 247=117, 120=103, 249=112, 121=103, 765=103, 254=110, 638=103, 510=103, 127=103, 895=103, 511=103}]]]] deadlock: false completed: 15189 Thread [name="sys-stripe-3-#4%null%", id=36, state=RUNNABLE, blockCnt=20, waitCnt=14908] at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at o.a.i.marshaller.jdk.JdkMarshallerInputStreamWrapper.read(JdkMarshallerInputStreamWrapper.java:53) at java.io.ObjectInputStream$PeekInputStream.read(ObjectInputStream.java:2320) at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2333) at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:2804) at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:802) at java.io.ObjectInputStream.<init>(ObjectInputStream.java:299) at o.a.i.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInputStream.java:39) at o.a.i.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:118) at o.a.i.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:94) at o.a.i.i.util.IgniteUtils.unmarshal(IgniteUtils.java:9752) at o.a.i.spi.discovery.tcp.TcpDiscoverySpi.readMessage(TcpDiscoverySpi.java:1467) at o.a.i.spi.discovery.tcp.ServerImpl.pingNode(ServerImpl.java:665) at o.a.i.spi.discovery.tcp.ServerImpl.pingNode(ServerImpl.java:565) at o.a.i.spi.discovery.tcp.ServerImpl.pingNode(ServerImpl.java:528) at o.a.i.spi.discovery.tcp.TcpDiscoverySpi.pingNode(TcpDiscoverySpi.java:439) at o.a.i.i.managers.discovery.GridDiscoveryManager.pingNode(GridDiscoveryManager.java:1417) at o.a.i.i.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:915) at o.a.i.i.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1059) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.map(GridDhtAtomicAbstractUpdateFuture.java:358) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1969) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1720) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateRequest(GridDhtAtomicCache.java:3179) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$1600(GridDhtAtomicCache.java:125) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:337) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:332) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:827) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:369) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:293) at o.a.i.i.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:95) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:238) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1215) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:843) at o.a.i.i.managers.communication.GridIoManager.access$2100(GridIoManager.java:108) at o.a.i.i.managers.communication.GridIoManager$6.run(GridIoManager.java:783) at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:428) at java.lang.Thread.run(Thread.java:745) -- Sent from: http://apache-ignite-users.70518.x6.nabble.com/