Hi, We uses Ignite 2.10 with 5 nodes, and we can see one of the nodes with the below warning message, and the cluster becomes unresponsive at the same time. May I know what's the possible reason for that, and how we can resolve the issue? Thanks.
thread=grid-timeout-worker-#22%Ignite% class=org.apache.ignite.internal.util.typedef.G >>> Possible starvation in striped pool. Thread name: sys-stripe-7-#8%Ignite% Queue: [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=BinaryObjectImpl [arr= true, ctx=false, start=0], val=BinaryObjectImpl [arr= true, ctx=false, start=0], prevVal=null, super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null, nearFutId=0, flags=keepBinary]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [keys=ArrayList [BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0]], vals=ArrayList [BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0]], prevVals=null, ttls=null, conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null, nearKeys=null, nearVals=null, obsoleteIndexes=null, forceTransformBackups=false, updateCntrs=GridLongList [idx=5, arr=[16537140,13196110,15013490,15241357,13947459]], super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null, nearFutId=0, flags=keepBinary]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[1475340]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[1475340]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [keys=ArrayList [BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0]], vals=ArrayList [BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=fal... and 2281 skipped ...[arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], Bin Deadlock: false Completed: 824135 Thread [name="sys-stripe-7-#8%Ignite%", id=20, state=TIMED_WAITING, blockCnt=127, waitCnt=7753416] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at o.a.i.i.processors.cache.persistence.pagemem.PagesWriteThrottle.onMarkDirty(PagesWriteThrottle.java:143) at o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlockPage(PageMemoryImpl.java:1738) at o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlock(PageMemoryImpl.java:505) at o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlock(PageMemoryImpl.java:497) at o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writeUnlock(PageHandler.java:401) at o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:308) at o.a.i.i.processors.cache.persistence.DataStructure.write(DataStructure.java:302) at o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:828) at o.a.i.i.processors.cache.persistence.RowStore.removeRow(RowStore.java:92) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishUpdate(IgniteCacheOffheapManagerImpl.java:2698) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.invoke0(IgniteCacheOffheapManagerImpl.java:1766) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.invoke(IgniteCacheOffheapManagerImpl.java:1741) at o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.invoke(GridCacheOffheapManager.java:2766) at o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.invoke(IgniteCacheOffheapManagerImpl.java:439) at o.a.i.i.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2338) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3403) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:146) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:325) at o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:320) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1908) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1529) at o.a.i.i.managers.communication.GridIoManager.access$5300(GridIoManager.java:242) at o.a.i.i.managers.communication.GridIoManager$9.execute(GridIoManager.java:1422) at o.a.i.i.managers.communication.TraceRunnable.run(TraceRunnable.java:55) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:569) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Regards, Marcus