Hi,
We uses Ignite 2.10 with 5 nodes, and we can see one of the nodes with the
below warning message, and the cluster becomes unresponsive at the same time.
May I know what's the possible reason for that, and how we can resolve the
issue? Thanks.
thread=grid-timeout-worker-#22%Ignite%
class=org.apache.ignite.internal.util.typedef.G >>> Possible starvation in
striped pool.
Thread name: sys-stripe-7-#8%Ignite%
Queue: [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE,
topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicSingleUpdateRequest [key=BinaryObjectImpl [arr= true,
ctx=false, start=0], val=BinaryObjectImpl [arr= true, ctx=false, start=0],
prevVal=null, super=GridDhtAtomicAbstractUpdateRequest [onRes=false,
nearNodeId=null, nearFutId=0, flags=keepBinary]]]], Message closure
[msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false,
timeout=0, skipOnTimeout=false, msg=GridDhtAtomicUpdateRequest [keys=ArrayList
[BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0]], vals=ArrayList [BinaryObjectImpl [arr= true, ctx=false,
start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl
[arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false,
start=0], BinaryObjectImpl [arr= true, ctx=false, start=0]], prevVals=null,
ttls=null, conflictExpireTimes=null, nearTtls=null, nearExpireTimes=null,
nearKeys=null, nearVals=null, obsoleteIndexes=null,
forceTransformBackups=false, updateCntrs=GridLongList [idx=5,
arr=[16537140,13196110,15013490,15241357,13947459]],
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=keepBinary]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1,
arr=[1475340]]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1,
arr=[1475340]]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicUpdateRequest [keys=ArrayList [BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0]],
vals=ArrayList [BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=false, start=0], BinaryObjectImpl [arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0],
BinaryObjectImpl [arr= true, ctx=fal... and 2281 skipped ...[arr= true,
ctx=false, start=0], BinaryObjectImpl [arr= true, ctx=false, start=0], Bin
Deadlock: false
Completed: 824135
Thread [name="sys-stripe-7-#8%Ignite%", id=20, state=TIMED_WAITING,
blockCnt=127, waitCnt=7753416]
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338)
at
o.a.i.i.processors.cache.persistence.pagemem.PagesWriteThrottle.onMarkDirty(PagesWriteThrottle.java:143)
at
o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlockPage(PageMemoryImpl.java:1738)
at
o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlock(PageMemoryImpl.java:505)
at
o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeUnlock(PageMemoryImpl.java:497)
at
o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writeUnlock(PageHandler.java:401)
at
o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:308)
at
o.a.i.i.processors.cache.persistence.DataStructure.write(DataStructure.java:302)
at
o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.removeDataRowByLink(AbstractFreeList.java:828)
at
o.a.i.i.processors.cache.persistence.RowStore.removeRow(RowStore.java:92)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.finishUpdate(IgniteCacheOffheapManagerImpl.java:2698)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.invoke0(IgniteCacheOffheapManagerImpl.java:1766)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.invoke(IgniteCacheOffheapManagerImpl.java:1741)
at
o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.invoke(GridCacheOffheapManager.java:2766)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.invoke(IgniteCacheOffheapManagerImpl.java:439)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2338)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3403)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:146)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:325)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:320)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1908)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1529)
at
o.a.i.i.managers.communication.GridIoManager.access$5300(GridIoManager.java:242)
at
o.a.i.i.managers.communication.GridIoManager$9.execute(GridIoManager.java:1422)
at
o.a.i.i.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:569)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
at java.lang.Thread.run(Thread.java:748)
Regards,
Marcus
