Just found a bunch of these in my logs as well. Note this is showing
starvation in the system threadpool, not the datastreamer threadpool, but
perhaps they're related?
[2018-06-28T17:39:55,728Z](grid-timeout-worker-#23)([]) WARN - G - >>>
Possible starvation in striped pool.
Thread name: sys-stripe-4-#5
Queue:
[o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@833da92,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@33e4268f,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@69c904f6,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@5b9aa1b6,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@4deb071d,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@5fa99071,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@7e66c1c6,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@707f48ad,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@65396a50,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@7600549e,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@3e20c369,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@3410de20,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@1ad55918,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@6e054a78,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@5606e75a,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@6455c264,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@54784a6f,
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@10ea9c12,
Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8,
ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=468,
val=null, hasValBytes=true], val=CacheObjectImpl [val=null,
hasValBytes=true], prevVal=null, super=GridDhtAtomicAbstractUpdateRequest
[onRes=false, nearNodeId=null, nearFutId=0, flags=]]]], Message closure
[msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false,
timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=468, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=468, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=468, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=492, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=132, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=132, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=132, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=132, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=132, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null,
nearFutId=0, flags=]]]], Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=K
Deadlock: false
Completed: 1010831
Thread [name="sys-stripe-4-#5", id=101, state=WAITING, blockCnt=596,
waitCnt=150343]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at
o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:177)
at
o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:140)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2799)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2621)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2585)
at
o.a.i.i.managers.communication.GridIoManager.send(GridIoManager.java:1642)
at
o.a.i.i.managers.communication.GridIoManager.sendToGridTopic(GridIoManager.java:1714)
at
o.a.i.i.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1166)
at
o.a.i.i.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1205)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.sendDeferredUpdateResponse(GridDhtAtomicCache.java:3375)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$3300(GridDhtAtomicCache.java:130)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout.run(GridDhtAtomicCache.java:3612)
at o.a.i.i.util.StripedExecutor$Stripe.run(StripedExecutor.java:505)
at java.lang.Thread.run(Thread.java:748)
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
