[
https://issues.apache.org/jira/browse/GEODE-6821?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16852457#comment-16852457
]
Barry Oglesby commented on GEODE-6821:
--------------------------------------
The key to this deadlock is a shared P2P message reader waiting in
SerialGatewaySenderQueue.put for a WriteLock like:
{noformat}
"P2P message reader for 192.168.1.2(server-3:54808)<v3>:41005 shared ordered
uid=6 port=62566" tid=0x4e owned by "Function Execution Processor7" tid=0x66
java.lang.Thread.State: WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on
java.util.concurrent.locks.ReentrantReadWriteLock$FairSync@42117a4e
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
at
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.put(SerialGatewaySenderQueue.java:220)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.queuePrimaryEvent(SerialGatewaySenderEventProcessor.java:477)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.enqueueEvent(SerialGatewaySenderEventProcessor.java:445)
at
org.apache.geode.internal.cache.wan.AbstractGatewaySender.distribute(AbstractGatewaySender.java:1033)
at
org.apache.geode.internal.cache.LocalRegion.notifyGatewaySender(LocalRegion.java:6138)
at
org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:5768)
at
org.apache.geode.internal.cache.map.RegionMapPut.doBeforeCompletionActions(RegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutAndDeliverEvent(AbstractRegionMapPut.java:301)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$163/1504099933.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWithIndexUpdatingInProgress(AbstractRegionMapPut.java:308)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutIfPreconditionsSatisified(AbstractRegionMapPut.java:296)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnSynchronizedRegionEntry(AbstractRegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnRegionEntryInMap(AbstractRegionMapPut.java:273)
- locked
org.apache.geode.internal.cache.entries.VersionedThinRegionEntryHeapIntKey@7bbbc992
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.addRegionEntryToMapAndDoPut(AbstractRegionMapPut.java:251)
- locked
org.apache.geode.internal.cache.entries.VersionedThinRegionEntryHeapIntKey@7bbbc992
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutRetryingIfNeeded(AbstractRegionMapPut.java:216)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$162/754294637.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doWithIndexInUpdateMode(AbstractRegionMapPut.java:198)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPut(AbstractRegionMapPut.java:180)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$161/453331027.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWhileLockedForCacheModification(AbstractRegionMapPut.java:119)
at
org.apache.geode.internal.cache.map.RegionMapPut.runWhileLockedForCacheModification(RegionMapPut.java:161)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.put(AbstractRegionMapPut.java:169)
at
org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2044)
at
org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5599)
at
org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:377)
at
org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:162)
at
org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5579)
at
org.apache.geode.internal.cache.AbstractUpdateOperation.doPutOrCreate(AbstractUpdateOperation.java:150)
at
org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.basicOperateOnRegion(AbstractUpdateOperation.java:285)
at
org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.operateOnRegion(AbstractUpdateOperation.java:256)
at
org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.basicProcess(DistributedCacheOperation.java:1200)
at
org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.process(DistributedCacheOperation.java:1100)
at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:369)
at
org.apache.geode.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:426)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.scheduleIncomingMessage(ClusterDistributionManager.java:2865)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.handleIncomingDMsg(ClusterDistributionManager.java:2552)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.access$1700(ClusterDistributionManager.java:111)
at
org.apache.geode.distributed.internal.ClusterDistributionManager$DMListener.messageReceived(ClusterDistributionManager.java:3395)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.dispatchMessage(GMSMembershipManager.java:1109)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.handleOrDeferMessage(GMSMembershipManager.java:1028)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager$MyDCReceiver.messageReceived(GMSMembershipManager.java:408)
at
org.apache.geode.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:702)
at
org.apache.geode.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:703)
at
org.apache.geode.internal.tcp.Connection.dispatchMessage(Connection.java:3393)
at org.apache.geode.internal.tcp.Connection.readMessage(Connection.java:3130)
at
org.apache.geode.internal.tcp.Connection.processInputBuffer(Connection.java:2925)
at org.apache.geode.internal.tcp.Connection.readMessages(Connection.java:1745)
at org.apache.geode.internal.tcp.Connection.run(Connection.java:1577)
at java.lang.Thread.run(Thread.java:745)
{noformat}
Another thread will have the WriteLock and be waitingForReplies like:
{noformat}
"Function Execution Processor7" tid=0x66
java.lang.Thread.State: TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
- waiting on java.util.concurrent.CountDownLatch$Sync@11c5d0ca
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at
org.apache.geode.internal.util.concurrent.StoppableCountDownLatch.await(StoppableCountDownLatch.java:71)
at
org.apache.geode.distributed.internal.ReplyProcessor21.basicWait(ReplyProcessor21.java:716)
at
org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:787)
at
org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:764)
at
org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:850)
at
org.apache.geode.internal.cache.DistributedCacheOperation.waitForAckIfNeeded(DistributedCacheOperation.java:770)
at
org.apache.geode.internal.cache.DistributedCacheOperation._distribute(DistributedCacheOperation.java:667)
at
org.apache.geode.internal.cache.DistributedCacheOperation.startOperation(DistributedCacheOperation.java:270)
at
org.apache.geode.internal.cache.DistributedCacheOperation.distribute(DistributedCacheOperation.java:311)
at
org.apache.geode.internal.cache.DistributedRegion.distributeUpdate(DistributedRegion.java:504)
at
org.apache.geode.internal.cache.DistributedRegion.basicPutPart3(DistributedRegion.java:482)
at
org.apache.geode.internal.cache.map.RegionMapPut.doAfterCompletionActions(RegionMapPut.java:307)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPut(AbstractRegionMapPut.java:185)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$161/453331027.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWhileLockedForCacheModification(AbstractRegionMapPut.java:119)
at
org.apache.geode.internal.cache.map.RegionMapPut.runWhileLockedForCacheModification(RegionMapPut.java:161)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.put(AbstractRegionMapPut.java:169)
at
org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2044)
at
org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5599)
at
org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:377)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$SerialGatewaySenderQueueMetaRegion.virtualPut(SerialGatewaySenderQueue.java:1208)
at
org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:162)
at org.apache.geode.internal.cache.LocalRegion.basicPut(LocalRegion.java:5025)
at
org.apache.geode.internal.cache.LocalRegion.validatedPut(LocalRegion.java:1617)
at org.apache.geode.internal.cache.LocalRegion.put(LocalRegion.java:1604)
at org.apache.geode.internal.cache.AbstractRegion.put(AbstractRegion.java:429)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.putAndGetKey(SerialGatewaySenderQueue.java:242)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.put(SerialGatewaySenderQueue.java:229)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.queuePrimaryEvent(SerialGatewaySenderEventProcessor.java:477)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.enqueueEvent(SerialGatewaySenderEventProcessor.java:445)
at
org.apache.geode.internal.cache.wan.AbstractGatewaySender.distribute(AbstractGatewaySender.java:1033)
at
org.apache.geode.internal.cache.LocalRegion.notifyGatewaySender(LocalRegion.java:6138)
at
org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:5768)
at
org.apache.geode.internal.cache.map.RegionMapPut.doBeforeCompletionActions(RegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutAndDeliverEvent(AbstractRegionMapPut.java:301)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$163/1504099933.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWithIndexUpdatingInProgress(AbstractRegionMapPut.java:308)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutIfPreconditionsSatisified(AbstractRegionMapPut.java:296)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnSynchronizedRegionEntry(AbstractRegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnRegionEntryInMap(AbstractRegionMapPut.java:273)
- locked
org.apache.geode.internal.cache.entries.VersionedThinRegionEntryHeapIntKey@12460465
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.addRegionEntryToMapAndDoPut(AbstractRegionMapPut.java:251)
- locked
org.apache.geode.internal.cache.entries.VersionedThinRegionEntryHeapIntKey@12460465
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutRetryingIfNeeded(AbstractRegionMapPut.java:216)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$162/754294637.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doWithIndexInUpdateMode(AbstractRegionMapPut.java:198)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPut(AbstractRegionMapPut.java:180)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut$$Lambda$161/453331027.run(Unknown
Source)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWhileLockedForCacheModification(AbstractRegionMapPut.java:119)
at
org.apache.geode.internal.cache.map.RegionMapPut.runWhileLockedForCacheModification(RegionMapPut.java:161)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.put(AbstractRegionMapPut.java:169)
at
org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2044)
at
org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5599)
at
org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:377)
at
org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:162)
at org.apache.geode.internal.cache.LocalRegion.basicPut(LocalRegion.java:5025)
at
org.apache.geode.internal.cache.LocalRegion.validatedPut(LocalRegion.java:1617)
at org.apache.geode.internal.cache.LocalRegion.put(LocalRegion.java:1604)
at org.apache.geode.internal.cache.AbstractRegion.put(AbstractRegion.java:429)
at
org.apache.geode.internal.cache.wan.asyncqueue.SerialAsyncEventListenersDifferentPrimariesDistributedTest$RegionOperationsFunction.doPut(SerialAsyncEventListenersDifferentPrimariesDistributedTest.java:274)
at
org.apache.geode.internal.cache.wan.asyncqueue.SerialAsyncEventListenersDifferentPrimariesDistributedTest$RegionOperationsFunction.execute(SerialAsyncEventListenersDifferentPrimariesDistributedTest.java:258)
at
org.apache.geode.internal.cache.execute.AbstractExecution.executeFunctionLocally(AbstractExecution.java:335)
at
org.apache.geode.internal.cache.execute.AbstractExecution$2.run(AbstractExecution.java:304)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.runUntilShutdown(ClusterDistributionManager.java:959)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.doFunctionExecutionThread(ClusterDistributionManager.java:813)
at
org.apache.geode.distributed.internal.ClusterDistributionManager$$Lambda$70/612638651.invoke(Unknown
Source)
at
org.apache.geode.internal.logging.LoggingThreadFactory.lambda$newThread$0(LoggingThreadFactory.java:121)
at
org.apache.geode.internal.logging.LoggingThreadFactory$$Lambda$60/1153076990.run(Unknown
Source)
at java.lang.Thread.run(Thread.java:745)
{noformat}
The member this Function Execution Processor thread is waiting for will be in
the exact same state.
> Multiple Serial GatewaySenders that are primary in different members can
> cause a distributed deadlock
> -----------------------------------------------------------------------------------------------------
>
> Key: GEODE-6821
> URL: https://issues.apache.org/jira/browse/GEODE-6821
> Project: Geode
> Issue Type: Bug
> Components: messaging, wan
> Reporter: Barry Oglesby
> Assignee: Barry Oglesby
> Priority: Major
>
> A test with this scenario causes a distributed deadlock.
> 3 servers each with:
> - a function that performs a random region operation on the input region
> - a replicated region on which the function is executed
> - two regions each with a serial AEQ (the type of region could be either
> replicate or partitioned)
> 1 multi-threaded client that repeatedly executes the function with random
> region names and operations.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
