[ 
https://issues.apache.org/jira/browse/GEODE-1894?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15490981#comment-15490981
 ] 

Hitesh Khamesra commented on GEODE-1894:
----------------------------------------

JNI global references: 6150

2016-09-14 09:33:59
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.66-b17 mixed mode):

"RMI TCP Connection(idle)" #10169 daemon prio=5 os_prio=0 
tid=0x00007f8f54003000 nid=0x2464 waiting on condition [0x00007f8fac9df000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0269958> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Attach Listener" #10071 daemon prio=9 os_prio=0 tid=0x00007f8fa8002000 
nid=0x1cea waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5364 daemon 
prio=5 os_prio=0 tid=0x00007f8f4c826800 nid=0xa12 waiting on condition 
[0x00007f8ed08bc000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0ef4128> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"AckReaderThread for : Event Processor for GatewaySender_ln.3" #5362 daemon 
prio=5 os_prio=0 tid=0x00007f8f18008800 nid=0xa11 runnable [0x00007f8ed0bbc000]
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:170)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at 
org.apache.geode.internal.cache.tier.sockets.Message.fetchHeader(Message.java:693)
        at 
org.apache.geode.internal.cache.tier.sockets.Message.readHeaderAndPayload(Message.java:710)
        at 
org.apache.geode.internal.cache.tier.sockets.Message.read(Message.java:661)
        at 
org.apache.geode.internal.cache.tier.sockets.Message.recv(Message.java:1103)
        - locked <0x00000000e0ffbdf8> (a java.nio.HeapByteBuffer)
        at 
org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attemptReadResponse(GatewaySenderBatchOp.java:205)
        at 
org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attemptRead(GatewaySenderBatchOp.java:166)
        at 
org.apache.geode.cache.client.internal.GatewaySenderBatchOp$GatewaySenderGFEBatchOpImpl.attempt(GatewaySenderBatchOp.java:145)
        at 
org.apache.geode.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:256)
        at 
org.apache.geode.cache.client.internal.pooling.PooledConnection.execute(PooledConnection.java:328)
        at 
org.apache.geode.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:937)
        at 
org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:579)
        at 
org.apache.geode.cache.client.internal.PoolImpl.executeOn(PoolImpl.java:748)
        at 
org.apache.geode.cache.client.internal.GatewaySenderBatchOp.executeOn(GatewaySenderBatchOp.java:72)
        at 
org.apache.geode.cache.client.internal.SenderProxy.receiveAckFromReceiver(SenderProxy.java:40)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.readAcknowledgement(GatewaySenderEventRemoteDispatcher.java:102)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher$AckReaderThread.run(GatewaySenderEventRemoteDispatcher.java:610)

   Locked ownable synchronizers:
        - None

"ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5363 daemon 
prio=5 os_prio=0 tid=0x00007f8f4c844800 nid=0xa10 waiting on condition 
[0x00007f8ed0cbe000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fe3178> (a 
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
        at 
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.destroyConnection(GatewaySenderEventRemoteDispatcher.java:343)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderEventRemoteDispatcher.stop(GatewaySenderEventRemoteDispatcher.java:798)
        at 
org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor.stopProcessing(AbstractGatewaySenderEventProcessor.java:1155)
        at 
org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor$SenderStopperCallable.call(AbstractGatewaySenderEventProcessor.java:1315)
        at 
org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor$SenderStopperCallable.call(AbstractGatewaySenderEventProcessor.java:1304)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - <0x00000000e0fe3440> (a 
java.util.concurrent.ThreadPoolExecutor$Worker)

"ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5361 daemon 
prio=5 os_prio=0 tid=0x00007f8f4c844000 nid=0xa0f waiting on condition 
[0x00007f8ed0dbf000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0ef4128> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5360 daemon 
prio=5 os_prio=0 tid=0x00007f8f4c843000 nid=0xa0e waiting on condition 
[0x00007f8ed0fc1000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0ef4128> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"ConcurrentSerialGatewaySenderEventProcessor Stopper Thread" #5359 daemon 
prio=5 os_prio=0 tid=0x00007f8f4c842800 nid=0xa0d waiting on condition 
[0x00007f8ed17c7000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0ef4128> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"ClientMembership Event Invoker" #5357 daemon prio=5 os_prio=0 
tid=0x00007f8f14002800 nid=0x9e7 waiting on condition [0x00007f8ed1ac8000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fdba28> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"poolTimer-ln-1" #5354 daemon prio=5 os_prio=0 tid=0x00007f8f14001800 nid=0x9e0 
waiting on condition [0x00007f8ed21cf000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fd9990> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Thread-1873" #5353 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a8800 nid=0x9d9 in 
Object.wait() [0x00007f8f308f2000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132)
        - locked <0x00000000e0fdf578> (a 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue)

   Locked ownable synchronizers:
        - None

"Thread-1872" #5351 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a7800 nid=0x9d8 in 
Object.wait() [0x00007f8f309f3000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132)
        - locked <0x00000000e0f533c8> (a 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue)

   Locked ownable synchronizers:
        - None

"Thread-1871" #5349 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a7000 nid=0x9d7 in 
Object.wait() [0x00007f8f30ef8000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132)
        - locked <0x00000000e0fe1560> (a 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue)

   Locked ownable synchronizers:
        - None

"Thread-1870" #5347 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a4800 nid=0x9d6 in 
Object.wait() [0x00007f8f30ff9000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132)
        - locked <0x00000000e0fdbb28> (a 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue)

   Locked ownable synchronizers:
        - None

"Thread-1869" #5345 daemon prio=5 os_prio=0 tid=0x00007f8f4c8a4000 nid=0x9d5 in 
Object.wait() [0x00007f8f312fc000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$BatchRemovalThread.run(SerialGatewaySenderQueue.java:1132)
        - locked <0x00000000e0feb0f8> (a 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue)

   Locked ownable synchronizers:
        - None

"Lock Grantor for org.apache.geode.internal.cache.wan.GatewaySenderAdvisor_ln" 
#5342 daemon prio=5 os_prio=0 tid=0x00007f8f4c848800 nid=0x9d4 in Object.wait() 
[0x00007f8fad1e3000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.distributed.internal.locks.DLockGrantor$DLockGrantorThread.run(DLockGrantor.java:3592)
        - locked <0x00000000e0fed0e0> (a java.lang.Object)

   Locked ownable synchronizers:
        - None

"Lock Grantor for gatewayEventIdIndexMetaData_lockService" #5338 daemon prio=5 
os_prio=0 tid=0x00007f8f4c847800 nid=0x9cb in Object.wait() [0x00007f8ed0ec0000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.distributed.internal.locks.DLockGrantor$DLockGrantorThread.run(DLockGrantor.java:3592)
        - locked <0x00000000e0fe15c8> (a java.lang.Object)

   Locked ownable synchronizers:
        - None

"Asynchronous disk writer for region ln" #5336 daemon prio=5 os_prio=0 
tid=0x00007f8f4c5b3800 nid=0x9c8 in Object.wait() [0x00007f8ed2eda000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:460)
        at java.util.concurrent.TimeUnit.timedWait(TimeUnit.java:348)
        at 
org.apache.geode.internal.cache.DiskStoreImpl$FlusherThread.waitUntilFlushIsReady(DiskStoreImpl.java:1648)
        - locked <0x00000000e0fdbb90> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.DiskStoreImpl$FlusherThread.run(DiskStoreImpl.java:1680)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"P2P message reader for 10.118.32.97(4449)<v4>:32777 shared unordered uid=225 
port=40671" #5333 daemon prio=10 os_prio=0 tid=0x00007f8eb8007800 nid=0x99d 
runnable [0x00007f8e0f1f2000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
        at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
        at sun.nio.ch.IOUtil.read(IOUtil.java:192)
        at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
        - locked <0x00000000e0fdf628> (a java.lang.Object)
        at 
org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823)
        at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"P2P message reader for 10.118.32.97(4344)<v2>:32775 shared ordered uid=234 
port=40668" #5332 daemon prio=10 os_prio=0 tid=0x00007f8eb8006800 nid=0x974 
runnable [0x00007f8facae0000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
        at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
        at sun.nio.ch.IOUtil.read(IOUtil.java:192)
        at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
        - locked <0x00000000e0fe1618> (a java.lang.Object)
        at 
org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823)
        at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"P2P message reader for 10.118.32.97(4396)<v3>:32776 shared unordered uid=225 
port=40662" #5330 daemon prio=10 os_prio=0 tid=0x00007f8eb8005800 nid=0x941 
runnable [0x00007f8e0f2f3000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
        at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
        at sun.nio.ch.IOUtil.read(IOUtil.java:192)
        at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
        - locked <0x00000000e0fef5c8> (a java.lang.Object)
        at 
org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823)
        at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Notification Handler" #5329 daemon prio=10 os_prio=0 tid=0x00007f8e4801e000 
nid=0x925 waiting on condition [0x00007f8e0fcfb000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fdf3b8> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Timer-171" #5327 daemon prio=5 os_prio=0 tid=0x00007f8ef4001800 nid=0x911 in 
Object.wait() [0x00007f8fad2e4000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        - locked <0x00000000e0ff3510> (a java.util.TaskQueue)
        at java.util.TimerThread.run(Timer.java:505)

   Locked ownable synchronizers:
        - None

"P2P message reader for 10.118.32.97(4344)<v2>:32775 shared unordered uid=231 
port=40646" #5326 daemon prio=10 os_prio=0 tid=0x00007f8eb8004800 nid=0x90f 
runnable [0x00007f8ed2cd8000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
        at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
        at sun.nio.ch.IOUtil.read(IOUtil.java:192)
        at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
        - locked <0x00000000e0feb190> (a java.lang.Object)
        at 
org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823)
        at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"View Message Processor" #5325 daemon prio=5 os_prio=0 tid=0x00007f8e34003800 
nid=0x90c waiting on condition [0x00007f8f307f1000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fdbbb0> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$3$1.run(DistributionManager.java:829)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Geode UDP INT-2,poland-54352" #5324 daemon prio=5 os_prio=0 
tid=0x00007f8e34002800 nid=0x909 waiting on condition [0x00007f8ed26d4000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0efd148> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Management Task" #5323 daemon prio=5 os_prio=0 tid=0x00007f8f4c7d4000 
nid=0x8f4 waiting on condition [0x00007f8ed13c3000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e11b7638> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at 
org.apache.geode.management.internal.LocalManager$1$1.run(LocalManager.java:128)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Pooled High Priority Message Processor 1" #5322 daemon prio=10 os_prio=0 
tid=0x00007f8de0001000 nid=0x8f3 waiting on condition [0x00007f8ed39fa000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fd9a58> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$5$1.run(DistributionManager.java:900)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"DiskStoreMonitor" #5321 daemon prio=5 os_prio=0 tid=0x00007f8f4c1df800 
nid=0x8f2 waiting on condition [0x00007f8f30cf6000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e1013e88> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Timer-170" #5320 daemon prio=5 os_prio=0 tid=0x00007f8f4c83b000 nid=0x8f1 in 
Object.wait() [0x00007f8e0f7f8000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        - locked <0x00000000e0fef750> (a java.util.TaskQueue)
        at java.util.TimerThread.run(Timer.java:505)

   Locked ownable synchronizers:
        - None

"Non-replicate Region Garbage Collector" #5319 daemon prio=5 os_prio=0 
tid=0x00007f8f4c7d7000 nid=0x8f0 in Object.wait() [0x00007f8ed38f9000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at 
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.doSleep(TombstoneService.java:871)
        - locked <0x00000000e0feb318> (a 
org.apache.geode.internal.cache.TombstoneService$NonReplicateTombstoneSweeper)
        at 
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:838)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Replicate/Partition Region Garbage Collector" #5318 daemon prio=5 os_prio=0 
tid=0x00007f8f4c879800 nid=0x8ef in Object.wait() [0x00007f8fac8de000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at 
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.doSleep(TombstoneService.java:871)
        - locked <0x00000000e0fd9ae0> (a 
org.apache.geode.internal.cache.TombstoneService$ReplicateTombstoneSweeper)
        at 
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:838)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Timer-169" #5317 daemon prio=5 os_prio=0 tid=0x00007f8f4c5a6000 nid=0x8ee in 
Object.wait() [0x00007f8ed30dc000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        - locked <0x00000000e0fed0f0> (a java.util.TaskQueue)
        at java.util.TimerThread.run(Timer.java:505)

   Locked ownable synchronizers:
        - None

"GlobalTXTimeoutMonitor" #5316 daemon prio=5 os_prio=0 tid=0x00007f8f4c82c800 
nid=0x8ed in Object.wait() [0x00007f8f30bf5000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.jta.TransactionManagerImpl$TransactionTimeOutThread.run(TransactionManagerImpl.java:711)
        - locked <0x00000000e0fdbcd0> (a 
java.util.Collections$SynchronizedSortedSet)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Thread-1854 StatSampler" #5314 daemon prio=10 os_prio=0 tid=0x00007f8f4c7fb000 
nid=0x8eb in Object.wait() [0x00007f8e0fefd000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at 
org.apache.geode.internal.statistics.HostStatSampler.delay(HostStatSampler.java:499)
        - locked <0x00000000e0eee5a0> (a 
org.apache.geode.internal.statistics.GemFireStatSampler)
        at 
org.apache.geode.internal.statistics.HostStatSampler.run(HostStatSampler.java:198)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"CallbackSampler" #5313 daemon prio=5 os_prio=0 tid=0x00007f8f4c7e2800 
nid=0x8ea waiting on condition [0x00007f8e0faf9000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0feb370> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"P2P message reader for 10.118.32.97(3769)<ec><v0>:32770 shared unordered 
uid=149 port=40643" #5312 daemon prio=10 os_prio=0 tid=0x00007f8eb8003800 
nid=0x8e9 runnable [0x00007f8ed3efd000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
        at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
        at sun.nio.ch.IOUtil.read(IOUtil.java:192)
        at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
        - locked <0x00000000e0fe17d0> (a java.lang.Object)
        at 
org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1823)
        at org.apache.geode.internal.tcp.Connection.run(Connection.java:1710)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"P2P-Handshaker poland.gemstone.com/10.118.32.97:0 Thread 0" #5311 daemon 
prio=10 os_prio=0 tid=0x00007f8f2c004800 nid=0x8e7 waiting on condition 
[0x00007f8ed02b8000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fef978> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
        at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"DM-MemberEventInvoker" #5297 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d4800 
nid=0x8df waiting on condition [0x00007f8ed12c2000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fd9b70> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.DistributionManager$MemberEventInvoker.run(DistributionManager.java:2392)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Geode UDP INT-1,poland-54352" #5308 daemon prio=5 os_prio=0 
tid=0x00007f8e34001800 nid=0x8dc waiting on condition [0x00007f8f310fa000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0efd148> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Geode Heartbeat Sender" #5307 daemon prio=5 os_prio=0 tid=0x00007f8f4c354800 
nid=0x8da waiting on condition [0x00007f8f311fb000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.sendPeriodicHeartbeats(GMSHealthMonitor.java:714)
        at 
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.run(GMSHealthMonitor.java:709)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - <0x00000000e0efa1e8> (a 
java.util.concurrent.ThreadPoolExecutor$Worker)

"Geode Failure Detection Server thread 0" #5306 daemon prio=5 os_prio=0 
tid=0x00007f8f4c87e000 nid=0x8d9 runnable [0x00007f8fac7dd000]
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainSocketImpl.socketAccept(Native Method)
        at 
java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
        at java.net.ServerSocket.implAccept(ServerSocket.java:545)
        at java.net.ServerSocket.accept(ServerSocket.java:513)
        at 
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$startTcpServer$16(GMSHealthMonitor.java:663)
        at 
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$7/1935064956.run(Unknown
 Source)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - <0x00000000e0f327e0> (a 
java.util.concurrent.ThreadPoolExecutor$Worker)

"P2P Listener Thread /10.118.32.97:36713" #5305 daemon prio=10 os_prio=0 
tid=0x00007f8f4c1e2800 nid=0x8d8 runnable [0x00007f8e0fbfa000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        - locked <0x00000000e0fed208> (a java.lang.Object)
        at org.apache.geode.internal.tcp.TCPConduit.run(TCPConduit.java:649)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Timer-168" #5304 daemon prio=5 os_prio=0 tid=0x00007f8f4c880000 nid=0x8d7 in 
Object.wait() [0x00007f8e0f4f5000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        - locked <0x00000000e0fd9c08> (a java.util.TaskQueue)
        at java.util.TimerThread.run(Timer.java:505)

   Locked ownable synchronizers:
        - None

"Geode Failure Detection Scheduler" #5303 daemon prio=5 os_prio=0 
tid=0x00007f8f4c8fb000 nid=0x8d6 waiting on condition [0x00007f8f305ef000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0f32558> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"unicast receiver,poland-54352" #5302 daemon prio=5 os_prio=0 
tid=0x00007f8f4c83a000 nid=0x8d5 runnable [0x00007f8f302ee000]
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainDatagramSocketImpl.receive0(Native Method)
        - locked <0x00000000e0f1f930> (a java.net.PlainDatagramSocketImpl)
        at 
java.net.AbstractPlainDatagramSocketImpl.receive(AbstractPlainDatagramSocketImpl.java:144)
        - locked <0x00000000e0f1f930> (a java.net.PlainDatagramSocketImpl)
        at java.net.DatagramSocket.receive(DatagramSocket.java:812)
        - locked <0x00000000e0fdbe08> (a java.net.DatagramPacket)
        - locked <0x00000000e0f1f8e0> (a java.net.MulticastSocket)
        at org.jgroups.protocols.UDP$PacketReceiver.run(UDP.java:694)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"TransferQueueBundler,poland-54352" #5301 daemon prio=5 os_prio=0 
tid=0x00007f8f4c1d1000 nid=0x8d4 waiting on condition [0x00007f8ed1bc9000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0eff380> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
        at 
org.jgroups.protocols.TransferQueueBundler.run(TransferQueueBundler.java:75)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Geode UDP Timer-2,poland-54352" #5300 daemon prio=5 os_prio=0 
tid=0x00007f8f4c81d800 nid=0x8d3 waiting on condition [0x00007f8ed3ffe000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0efe350> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Timer runner-1,poland-54352" #5299 daemon prio=5 os_prio=0 
tid=0x00007f8f4c5a3800 nid=0x8d2 waiting on condition [0x00007f8ed24d2000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0efef88> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.DelayQueue.take(DelayQueue.java:223)
        at java.util.concurrent.DelayQueue.take(DelayQueue.java:70)
        at org.jgroups.util.TimeScheduler3.run(TimeScheduler3.java:166)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Geode Membership Timer" #5298 daemon prio=5 os_prio=0 tid=0x00007f8f4c1d6000 
nid=0x8d1 in Object.wait() [0x00007f8ed2dd9000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        - locked <0x00000000e0f31548> (a java.util.TaskQueue)
        at java.util.TimerThread.run(Timer.java:505)

   Locked ownable synchronizers:
        - None

"Function Execution Processor0" #5296 daemon prio=5 os_prio=0 
tid=0x00007f8f4c5b1800 nid=0x8d0 waiting on condition [0x00007f8ed01b7000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0ff3570> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
org.apache.geode.distributed.internal.FunctionExecutionPooledExecutor$2.run(FunctionExecutionPooledExecutor.java:143)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1049)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"PartitionedRegion Message Processor0" #5295 daemon prio=5 os_prio=0 
tid=0x00007f8f4c1d2800 nid=0x8cf waiting on condition [0x00007f8ed16c4000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fef3c8> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$8$1.run(DistributionManager.java:1006)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"PrMetaData cleanup Message Processor 0" #5294 daemon prio=5 os_prio=0 
tid=0x00007f8f4c885000 nid=0x8ce waiting on condition [0x00007f8ed34e0000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0feb438> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$7$1.run(DistributionManager.java:967)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Pooled High Priority Message Processor 0" #5293 daemon prio=5 os_prio=0 
tid=0x00007f8f4c83c000 nid=0x8cd waiting on condition [0x00007f8f30af4000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fd9c20> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$5$1.run(DistributionManager.java:900)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Pooled Message Processor 0" #5292 daemon prio=5 os_prio=0 
tid=0x00007f8f4c88a000 nid=0x8cc waiting on condition [0x00007f8e0fdfc000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e100e968> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
org.apache.geode.distributed.internal.OverflowQueueWithDMStats.take(OverflowQueueWithDMStats.java:104)
        at 
org.apache.geode.distributed.internal.PooledExecutorWithDMStats$1.run(PooledExecutorWithDMStats.java:92)
        at 
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:604)
        at 
org.apache.geode.distributed.internal.DistributionManager$4$1.run(DistributionManager.java:863)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Queued Gateway Listener Thread" #4223 daemon prio=10 os_prio=0 
tid=0x00007f8e3c00a800 nid=0x70eb waiting on condition [0x00007f8ed28d6000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0d905e8> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln.4" #4220 daemon prio=5 os_prio=0 
tid=0x00007f8ed8006800 nid=0x70ea in Object.wait() [0x00007f8f301ed000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486)
        - locked <0x00000000e0d89fe8> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln.3" #4218 daemon prio=5 os_prio=0 
tid=0x00007f8ed8005000 nid=0x70e9 in Object.wait() [0x00007f8ed2bd7000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486)
        - locked <0x00000000e0d89fe8> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln.2" #4214 daemon prio=5 os_prio=0 
tid=0x00007f8ed8004000 nid=0x70e8 in Object.wait() [0x00007f8ed3dfc000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486)
        - locked <0x00000000e0d89fe8> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln.1" #4211 daemon prio=5 os_prio=0 
tid=0x00007f8ed8001800 nid=0x70e7 in Object.wait() [0x00007f8ed1cca000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486)
        - locked <0x00000000e0d89fe8> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln.0" #4208 daemon prio=5 os_prio=0 
tid=0x00007f8ed8002800 nid=0x70e6 in Object.wait() [0x00007f8ed1fcd000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at 
org.apache.geode.internal.cache.wan.GatewaySenderAdvisor.waitToBecomePrimary(GatewaySenderAdvisor.java:486)
        - locked <0x00000000e0d89fe8> (a java.lang.Object)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.waitForPrimary(SerialGatewaySenderEventProcessor.java:154)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:203)

   Locked ownable synchronizers:
        - None

"Event Processor for GatewaySender_ln" #4207 daemon prio=5 os_prio=0 
tid=0x00007f8f00009800 nid=0x70e5 in Object.wait() [0x00007f8ed1ecc000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1245)
        - locked <0x00000000e0d8be80> (a 
org.apache.geode.internal.cache.wan.serial.RemoteSerialGatewaySenderEventProcessor)
        at java.lang.Thread.join(Thread.java:1319)
        at 
org.apache.geode.internal.cache.wan.serial.ConcurrentSerialGatewaySenderEventProcessor.run(ConcurrentSerialGatewaySenderEventProcessor.java:190)

   Locked ownable synchronizers:
        - None

"Queued Gateway Listener Thread" #4221 daemon prio=10 os_prio=0 
tid=0x00007f8e3c003800 nid=0x70e3 waiting on condition [0x00007f8ed22d0000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0d90860> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Queued Gateway Listener Thread" #4217 daemon prio=10 os_prio=0 
tid=0x00007f8e3c006800 nid=0x70e1 waiting on condition [0x00007f8ed25d3000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0d92930> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Queued Gateway Listener Thread" #4215 daemon prio=10 os_prio=0 
tid=0x00007f8e3c002800 nid=0x70df waiting on condition [0x00007f8f30df7000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0d92af0> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Queued Gateway Listener Thread" #4212 daemon prio=10 os_prio=0 
tid=0x00007f8e3c009800 nid=0x70dd waiting on condition [0x00007f8facce2000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0d93cc0> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"RMI TCP Connection(1)-10.118.32.97" #27 daemon prio=5 os_prio=0 
tid=0x00007f8f54001800 nid=0x10f7 waiting on condition [0x00007f8fad7e2000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e0fe3420> (a 
java.util.concurrent.FutureTask)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)
        at java.util.concurrent.FutureTask.get(FutureTask.java:191)
        at 
java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:244)
        at 
org.apache.geode.internal.cache.wan.serial.ConcurrentSerialGatewaySenderEventProcessor.stopProcessing(ConcurrentSerialGatewaySenderEventProcessor.java:296)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderImpl.stop(SerialGatewaySenderImpl.java:134)
        at 
org.apache.geode.internal.cache.wan.WANTestBase.stopSender(WANTestBase.java:1635)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.lambda$testStopOneSerialGatewaySenderBothPrimary$bb17a952$2(SerialGatewaySenderOperationsDUnitTest.java:303)
        at 
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$141/481914353.run(Unknown
 Source)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:497)
        at hydra.MethExecutor.executeObject(MethExecutor.java:268)
        at 
org.apache.geode.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:82)
        at sun.reflect.GeneratedMethodAccessor309.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:497)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
        at sun.rmi.transport.Transport$1.run(Transport.java:200)
        at sun.rmi.transport.Transport$1.run(Transport.java:197)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
        at 
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$1/385178200.run(Unknown
 Source)
        at java.security.AccessController.doPrivileged(Native Method)
        at 
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - <0x00000000e030d318> (a 
java.util.concurrent.ThreadPoolExecutor$Worker)
        - <0x00000000e11b70b0> (a 
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)

"RMI Reaper" #26 prio=5 os_prio=0 tid=0x00007f90207f5800 nid=0x10f5 in 
Object.wait() [0x00007f8fad8e6000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        - waiting on <0x00000000e02d4a18> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
        - locked <0x00000000e02d4a18> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
        at sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:351)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007f90207f1000 
nid=0x10f4 runnable [0x00007f8fad9e7000]
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainSocketImpl.socketAccept(Native Method)
        at 
java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
        at java.net.ServerSocket.implAccept(ServerSocket.java:545)
        at java.net.ServerSocket.accept(ServerSocket.java:513)
        at 
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
        at 
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"GC Daemon" #24 daemon prio=2 os_prio=0 tid=0x00007f90207e0000 nid=0x10f2 in 
Object.wait() [0x00007f8fadae8000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at sun.misc.GC$Daemon.run(GC.java:117)
        - locked <0x00000000e001de88> (a sun.misc.GC$LatencyLock)

   Locked ownable synchronizers:
        - None

"RMI RenewClean-[10.118.32.97:51910]" #23 daemon prio=5 os_prio=0 
tid=0x00007f90207de800 nid=0x10f1 in Object.wait() [0x00007f8fadbe9000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
        - locked <0x00000000e0025418> (a java.lang.ref.ReferenceQueue$Lock)
        at 
sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:553)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"RMI Scheduler(0)" #22 daemon prio=5 os_prio=0 tid=0x00007f90207dc000 
nid=0x10f0 waiting on condition [0x00007f8fadcea000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000e005feb8> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

   Locked ownable synchronizers:
        - None

"Service Thread" #19 daemon prio=9 os_prio=0 tid=0x00007f9020116000 nid=0x10ec 
runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C1 CompilerThread11" #18 daemon prio=9 os_prio=0 tid=0x00007f9020111000 
nid=0x10eb waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C1 CompilerThread10" #17 daemon prio=9 os_prio=0 tid=0x00007f902010e800 
nid=0x10ea waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C1 CompilerThread9" #16 daemon prio=9 os_prio=0 tid=0x00007f902010c800 
nid=0x10e9 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C1 CompilerThread8" #15 daemon prio=9 os_prio=0 tid=0x00007f902010a000 
nid=0x10e8 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread7" #14 daemon prio=9 os_prio=0 tid=0x00007f9020108000 
nid=0x10e7 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread6" #13 daemon prio=9 os_prio=0 tid=0x00007f9020105800 
nid=0x10e6 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread5" #12 daemon prio=9 os_prio=0 tid=0x00007f9020103800 
nid=0x10e5 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread4" #11 daemon prio=9 os_prio=0 tid=0x00007f90200f9800 
nid=0x10e4 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread3" #10 daemon prio=9 os_prio=0 tid=0x00007f90200f7000 
nid=0x10e3 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread2" #9 daemon prio=9 os_prio=0 tid=0x00007f90200f5000 
nid=0x10e2 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread1" #8 daemon prio=9 os_prio=0 tid=0x00007f90200f3000 
nid=0x10e1 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"C2 CompilerThread0" #7 daemon prio=9 os_prio=0 tid=0x00007f90200f0000 
nid=0x10e0 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"JDWP Event Helper Thread" #6 daemon prio=10 os_prio=0 tid=0x00007f90200ed800 
nid=0x10df runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"JDWP Transport Listener: dt_socket" #5 daemon prio=10 os_prio=0 
tid=0x00007f90200ea000 nid=0x10de runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f90200da800 
nid=0x10dd runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f90200a3000 nid=0x10dc in 
Object.wait() [0x00007f8fbc4d6000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
        - locked <0x00000000e0013e50> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
        at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

   Locked ownable synchronizers:
        - None

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f90200a1000 
nid=0x10db in Object.wait() [0x00007f8fbc5d7000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
        - locked <0x00000000e0014008> (a java.lang.ref.Reference$Lock)

   Locked ownable synchronizers:
        - None

"main" #1 prio=5 os_prio=0 tid=0x00007f9020010000 nid=0x10c7 waiting on 
condition [0x00007f9027646000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
        at java.lang.Thread.sleep(Native Method)
        at org.apache.geode.test.dunit.standalone.ChildVM.main(ChildVM.java:58)

   Locked ownable synchronizers:
        - None

"VM Thread" os_prio=0 tid=0x00007f902009c000 nid=0x10da runnable 

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f9020025800 nid=0x10c8 
runnable 

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f9020027800 nid=0x10c9 
runnable 

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f9020029000 nid=0x10ca 
runnable 

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f902002b000 nid=0x10cb 
runnable 

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f902002d000 nid=0x10cc 
runnable 

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f902002e800 nid=0x10cd 
runnable 

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f9020030800 nid=0x10ce 
runnable 

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f9020032000 nid=0x10cf 
runnable 

"GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007f9020034000 nid=0x10d0 
runnable 

"GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007f9020036000 nid=0x10d1 
runnable 

"GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007f9020037800 nid=0x10d2 
runnable 

"GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007f9020039800 nid=0x10d3 
runnable 

"GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007f902003b800 nid=0x10d4 
runnable 

"GC task thread#13 (ParallelGC)" os_prio=0 tid=0x00007f902003d000 nid=0x10d5 
runnable 

"GC task thread#14 (ParallelGC)" os_prio=0 tid=0x00007f902003f000 nid=0x10d6 
runnable 

"GC task thread#15 (ParallelGC)" os_prio=0 tid=0x00007f9020041000 nid=0x10d7 
runnable 

"GC task thread#16 (ParallelGC)" os_prio=0 tid=0x00007f9020042800 nid=0x10d8 
runnable 

"GC task thread#17 (ParallelGC)" os_prio=0 tid=0x00007f9020044800 nid=0x10d9 
runnable 

"VM Periodic Task Thread" os_prio=0 tid=0x00007f9020119800 nid=0x10ed waiting 
on condition 

JNI global references: 6409

2016-09-14 09:33:59
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.66-b17 mixed mode):


> SerialGatewaySenderOperationsDUnitTest test hangs
> -------------------------------------------------
>
>                 Key: GEODE-1894
>                 URL: https://issues.apache.org/jira/browse/GEODE-1894
>             Project: Geode
>          Issue Type: Bug
>          Components: wan
>            Reporter: Hitesh Khamesra
>
> test tries to stop Serial Gateway Sender and that thread just hangs. Event 
> processors are waiting to become primary. One AckReader thread waiting for 
> ack. Seems like need to interrupt these threads. Attached thread dump 



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Reply via email to