I would recommend you to upgrade to 2.14 this is likely fixed already.
On Tue, Jul 21, 2020 at 5:58 AM Олег АндреичЪ <[email protected]> wrote: > > Hello again, > > Here's a much simpler case involving only two threads: > --- > Found one Java-level deadlock: > ============================= > "Thread-2 (ActiveMQ-client-global-threads)": > waiting for ownable synchronizer 0x000000067438fc48, (a > java.util.concurrent.locks.ReentrantLock$NonfairSync), > which is held by "ExecScriptAction #28" > "ExecScriptAction #28": > waiting for ownable synchronizer 0x00000006744d7e40, (a > java.util.concurrent.locks.ReentrantLock$NonfairSync), > which is held by "Thread-2 (ActiveMQ-client-global-threads)" > > Java stack information for the threads listed above: > =================================================== > "Thread-2 (ActiveMQ-client-global-threads)": > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0x000000067438fc48> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:837) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:872) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1202) > at > java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:213) > at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.returnBlocking(ChannelImpl.java:216) > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.returnBlocking(ActiveMQSessionContext.java:250) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.handleFailover(ClientSessionImpl.java:1428) > - locked <0x00000006744cbb08> (a > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.reconnectSessions(ClientSessionFactoryImpl.java:774) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:617) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:500) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$100(ClientSessionFactoryImpl.java:73) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$1.run(ClientSessionFactoryImpl.java:363) > at > org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:42) > at > org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:31) > at > org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:66) > at > org.apache.activemq.artemis.utils.actors.ProcessorBase$$Lambda$420/1534144616.run(Unknown > Source) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at > org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:118) > "ExecScriptAction #28": > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0x00000006744d7e40> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:837) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:872) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1202) > at > java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:213) > at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.lockFailover(ClientSessionFactoryImpl.java:233) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:543) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$600(ClientSessionFactoryImpl.java:73) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingFailureListener.connectionFailed(ClientSessionFactoryImpl.java:1222) > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.callFailureListeners(AbstractRemotingConnection.java:75) > at > org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:219) > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.fail(AbstractRemotingConnection.java:218) > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:455) > - locked <0x00000006744cc968> (a java.lang.Object) > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.simpleCommit(ActiveMQSessionContext.java:455) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.commit(ClientSessionImpl.java:939) > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.commit(ClientSessionImpl.java:912) > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.commit(ActiveMQSession.java:288) > at > org.springframework.jms.connection.JmsResourceHolder.commitAll(JmsResourceHolder.java:184) > at > org.springframework.jms.connection.ConnectionFactoryUtils$JmsResourceSynchronization.processResourceAfterCommit(ConnectionFactoryUtils.java:421) > at > org.springframework.jms.connection.ConnectionFactoryUtils$JmsResourceSynchronization.processResourceAfterCommit(ConnectionFactoryUtils.java:404) > at > org.springframework.transaction.support.ResourceHolderSynchronization.afterCommit(ResourceHolderSynchronization.java:85) > at > org.springframework.transaction.support.TransactionSynchronizationUtils.invokeAfterCommit(TransactionSynchronizationUtils.java:133) > at > org.springframework.transaction.support.TransactionSynchronizationUtils.triggerAfterCommit(TransactionSynchronizationUtils.java:121) > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.triggerAfterCommit(AbstractPlatformTransactionManager.java:958) > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:803) > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730) > at > org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:150) > ... > > Found 1 deadlock. > --- > -- > Oleg Kleshchev > > On Fri, Jul 17, 2020 at 4:11 PM Олег АндреичЪ <[email protected]> wrote: > > > > Hello, > > > > Some time ago I faced a deadlock in the connection/failover code in the > Artemis client (ver. 2.9.0) under load. > > Could you please give me any ideas on how to resolve this issue? > > > > The client is a spring application that receives and sends JMS messages > to the broker. > > > > Here are relevant thread dumps with insignificant parts omitted: > > > > The first one is with apparent deadlock. > > --- > > Found one Java-level deadlock: > > ============================= > > "EventActionMessageContainer-8": > > waiting to lock monitor 0x00007f0698094228 (object 0x00000006594e8680, > a java.lang.Object), > > which is held by "EventActionMessageContainer-4" > > "EventActionMessageContainer-4": > > waiting for ownable synchronizer 0x00000006597cf738, (a > java.util.concurrent.locks.ReentrantLock$NonfairSync), > > which is held by "Thread-0 (ActiveMQ-client-global-threads)" > > "Thread-0 (ActiveMQ-client-global-threads)": > > waiting for ownable synchronizer 0x00000006594e8620, (a > java.util.concurrent.locks.ReentrantLock$NonfairSync), > > which is held by "EventActionMessageContainer-4" > > > > Java stack information for the threads listed above: > > =================================================== > > "EventActionMessageContainer-8": > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:396) > > - waiting to lock <0x00000006594e8680> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.sessionClose(ActiveMQSessionContext.java:619) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.close(ClientSessionImpl.java:1311) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.closeCleanSessions(ClientSessionFactoryImpl.java:445) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.interruptConnectAndCloseAllSessions(ClientSessionFactoryImpl.java:427) > > - locked <0x00000006597cf718> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.close(ClientSessionFactoryImpl.java:460) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnection.close(ActiveMQConnection.java:356) > > - locked <0x0000000659380dc0> (a > org.apache.activemq.artemis.jms.client.ActiveMQConnection) > > at > org.springframework.jms.connection.ConnectionFactoryUtils.releaseConnection(ConnectionFactoryUtils.java:80) > > at > org.springframework.jms.listener.AbstractJmsListeningContainer.refreshSharedConnection(AbstractJmsListeningContainer.java:395) > > - locked <0x00000006593a0b28> (a java.lang.Object) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer.refreshConnectionUntilSuccessful(DefaultMessageListenerContainer.java:925) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer.recoverAfterListenerSetupFailure(DefaultMessageListenerContainer.java:899) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1077) > > - locked <0x0000000659659158> (a java.lang.Object) > > at java.lang.Thread.run(Thread.java:748) > > "EventActionMessageContainer-4": > > at sun.misc.Unsafe.park(Native Method) > > - parking to wait for <0x00000006597cf738> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:837) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:872) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1202) > > at > java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:213) > > at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.lockFailover(ClientSessionFactoryImpl.java:233) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:543) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$600(ClientSessionFactoryImpl.java:73) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingFailureListener.connectionFailed(ClientSessionFactoryImpl.java:1222) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.callFailureListeners(AbstractRemotingConnection.java:75) > > at > org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:219) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.fail(AbstractRemotingConnection.java:218) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:455) > > - locked <0x00000006594e8680> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.createConsumer(ActiveMQSessionContext.java:394) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.internalCreateConsumer(ClientSessionImpl.java:1963) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.createConsumer(ClientSessionImpl.java:859) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.createConsumer(ClientSessionImpl.java:807) > > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.createClientConsumer(ActiveMQSession.java:921) > > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.createConsumer(ActiveMQSession.java:816) > > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.createConsumer(ActiveMQSession.java:462) > > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.createConsumer(ActiveMQSession.java:440) > > at > org.springframework.jms.listener.AbstractMessageListenerContainer.createConsumer(AbstractMessageListenerContainer.java:881) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.createListenerConsumer(AbstractPollingMessageListenerContainer.java:215) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:299) > > at > core.server.jms.BackGroundClusterNodeListenerContainer.doReceiveAndExecute(BackGroundClusterNodeListenerContainer.java:80) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:255) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1166) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1158) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1055) > > at java.lang.Thread.run(Thread.java:748) > > "Thread-0 (ActiveMQ-client-global-threads)": > > at sun.misc.Unsafe.park(Native Method) > > - parking to wait for <0x00000006594e8620> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:837) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:872) > > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1202) > > at > java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:213) > > at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.lock(ChannelImpl.java:603) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.lockCommunications(ActiveMQSessionContext.java:255) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.preHandleFailover(ClientSessionImpl.java:1348) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.reconnectSessions(ClientSessionFactoryImpl.java:744) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:617) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:500) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$100(ClientSessionFactoryImpl.java:73) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$1.run(ClientSessionFactoryImpl.java:363) > > at > org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:42) > > at > org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:31) > > at > org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:66) > > at > org.apache.activemq.artemis.utils.actors.ProcessorBase$$Lambda$420/42633625.run(Unknown > Source) > > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > > at > org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:118) > > > > Found 1 deadlock. > > --- > > > > Another one with no obvious deadlock (due to the asynchronous nature of > the client), but stuck nonetheless. > > --- > > "ExecScriptAction #27" #27 daemon prio=5 os_prio=0 tid=0x00007fb468725000 > nid=0x7f0c0 in Object.wait() [0x00007fb3d77a2000] > > java.lang.Thread.State: WAITING (on object monitor) > > at java.lang.Object.wait(Native Method) > > at java.lang.Object.wait(Object.java:502) > > at > io.netty.util.concurrent.DefaultPromise.awaitUninterruptibly(DefaultPromise.java:243) > > - locked <0x00000007930f3268> (a > io.netty.bootstrap.AbstractBootstrap$PendingRegistrationPromise) > > at > io.netty.channel.DefaultChannelPromise.awaitUninterruptibly(DefaultChannelPromise.java:137) > > at > io.netty.channel.DefaultChannelPromise.awaitUninterruptibly(DefaultChannelPromise.java:30) > > at > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector.createConnection(NettyConnector.java:778) > > at > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector.createConnection(NettyConnector.java:740) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.openTransportConnection(ClientSessionFactoryImpl.java:1051) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.createTransportConnection(ClientSessionFactoryImpl.java:1100) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.establishNewConnection(ClientSessionFactoryImpl.java:1311) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.getConnection(ClientSessionFactoryImpl.java:906) > > - locked <0x00000007930ef8e0> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.getConnectionWithRetry(ClientSessionFactoryImpl.java:802) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.connect(ClientSessionFactoryImpl.java:241) > > at > org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:780) > > - locked <0x000000064aa73d58> (a > org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory.createConnectionInternal(ActiveMQConnectionFactory.java:844) > > - locked <0x000000064aa73d20> (a > org.apache.activemq.artemis.jms.client.ActiveMQJMSConnectionFactory) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:282) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:277) > > at > org.springframework.jms.support.JmsAccessor.createConnection(JmsAccessor.java:180) > > at > org.springframework.jms.core.JmsTemplate.access$600(JmsTemplate.java:90) > > at > org.springframework.jms.core.JmsTemplate$JmsTemplateResourceFactory.createConnection(JmsTemplate.java:1177) > > at > org.springframework.jms.connection.ConnectionFactoryUtils.doGetTransactionalSession(ConnectionFactoryUtils.java:312) > > at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:471) > > at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:570) > > ... > > > > Locked ownable synchronizers: > > - <0x0000000793657308> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > - <0x0000000793665888> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > - <0x00000006400b51e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) > > > > "EventActionMessageContainer-7" #275 prio=5 os_prio=0 > tid=0x00007fb394a0d800 nid=0x7fbac waiting for monitor entry > [0x00007fb33a4b9000] > > java.lang.Thread.State: BLOCKED (on object monitor) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:397) > > - waiting to lock <0x000000065cd70a80> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.sessionClose(ActiveMQSessionContext.java:619) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.close(ClientSessionImpl.java:1311) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.closeCleanSessions(ClientSessionFactoryImpl.java:445) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.interruptConnectAndCloseAllSessions(ClientSessionFactoryImpl.java:427) > > - locked <0x000000065ced93d8> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.close(ClientSessionFactoryImpl.java:460) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnection.close(ActiveMQConnection.java:356) > > - locked <0x000000065c838aa8> (a > org.apache.activemq.artemis.jms.client.ActiveMQConnection) > > at > org.springframework.jms.connection.ConnectionFactoryUtils.releaseConnection(ConnectionFactoryUtils.java:80) > > at > org.springframework.jms.listener.AbstractJmsListeningContainer.refreshSharedConnection(AbstractJmsListeningContainer.java:395) > > - locked <0x000000065c8f32b8> (a java.lang.Object) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer.refreshConnectionUntilSuccessful(DefaultMessageListenerContainer.java:925) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer.recoverAfterListenerSetupFailure(DefaultMessageListenerContainer.java:899) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1077) > > - locked <0x000000065c8f4610> (a java.lang.Object) > > at java.lang.Thread.run(Thread.java:748) > > > > Locked ownable synchronizers: > > - None > > > > "Reports (Complete)-1" #292 prio=5 os_prio=0 tid=0x00007fb395a68000 > nid=0x7fbd3 waiting for monitor entry [0x00007fb3392a6000] > > java.lang.Thread.State: BLOCKED (on object monitor) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:396) > > - waiting to lock <0x0000000794a5c028> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.sessionStop(ActiveMQSessionContext.java:473) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.stop(ClientSessionImpl.java:1096) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.stop(ClientSessionImpl.java:1085) > > at > org.apache.activemq.artemis.jms.client.ActiveMQSession.stop(ActiveMQSession.java:1214) > > at > org.apache.activemq.artemis.jms.client.ActiveMQConnection.stop(ActiveMQConnection.java:341) > > - locked <0x000000065d080678> (a > org.apache.activemq.artemis.jms.client.ActiveMQConnection) > > at > org.springframework.jms.connection.SingleConnectionFactory$SharedConnectionInvocationHandler.localStop(SingleConnectionFactory.java:644) > > - locked <0x0000000794a61f20> (a java.lang.Object) > > at > org.springframework.jms.connection.SingleConnectionFactory$SharedConnectionInvocationHandler.invoke(SingleConnectionFactory.java:577) > > at com.sun.proxy.$Proxy311.close(Unknown Source) > > at > org.springframework.jms.connection.ConnectionFactoryUtils.releaseConnection(ConnectionFactoryUtils.java:80) > > at > org.springframework.jms.connection.JmsResourceHolder.closeAll(JmsResourceHolder.java:232) > > at > org.springframework.jms.connection.ConnectionFactoryUtils$JmsResourceSynchronization.releaseResource(ConnectionFactoryUtils.java:430) > > at > org.springframework.jms.connection.ConnectionFactoryUtils$JmsResourceSynchronization.releaseResource(ConnectionFactoryUtils.java:404) > > at > org.springframework.transaction.support.ResourceHolderSynchronization.beforeCompletion(ResourceHolderSynchronization.java:77) > > at > org.springframework.transaction.support.TransactionSynchronizationUtils.triggerBeforeCompletion(TransactionSynchronizationUtils.java:106) > > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.triggerBeforeCompletion(AbstractPlatformTransactionManager.java:945) > > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:745) > > at > org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:730) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:249) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1166) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1158) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1055) > > at java.lang.Thread.run(Thread.java:748) > > > > Locked ownable synchronizers: > > - None > > > > "EventActionMessageContainer-8" #276 prio=5 os_prio=0 > tid=0x00007fb394a0f000 nid=0x7fbad in Object.wait() [0x00007fb33a3b7000] > > java.lang.Thread.State: WAITING (on object monitor) > > at java.lang.Object.wait(Native Method) > > at java.lang.Object.wait(Object.java:502) > > at > io.netty.util.concurrent.DefaultPromise.awaitUninterruptibly(DefaultPromise.java:243) > > - locked <0x0000000794af23a0> (a > io.netty.channel.group.DefaultChannelGroupFuture) > > at > io.netty.channel.group.DefaultChannelGroupFuture.awaitUninterruptibly(DefaultChannelGroupFuture.java:183) > > at > io.netty.channel.group.DefaultChannelGroupFuture.awaitUninterruptibly(DefaultChannelGroupFuture.java:40) > > at > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector.close(NettyConnector.java:718) > > - locked <0x000000065c8b5678> (a > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:607) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$600(ClientSessionFactoryImpl.java:73) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingFailureListener.connectionFailed(ClientSessionFactoryImpl.java:1222) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.callFailureListeners(AbstractRemotingConnection.java:75) > > at > org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:219) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.fail(AbstractRemotingConnection.java:218) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:455) > > - locked <0x000000065cd70a80> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.closeConsumer(ActiveMQSessionContext.java:433) > > at > org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.doCleanUp(ClientConsumerImpl.java:1113) > > at > org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.close(ClientConsumerImpl.java:462) > > at > org.apache.activemq.artemis.jms.client.ActiveMQMessageConsumer.close(ActiveMQMessageConsumer.java:145) > > at > org.springframework.jms.support.JmsUtils.closeMessageConsumer(JmsUtils.java:151) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:356) > > at > core.server.jms.BackGroundClusterNodeListenerContainer.doReceiveAndExecute(BackGroundClusterNodeListenerContainer.java:80) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:255) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1166) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1158) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1055) > > at java.lang.Thread.run(Thread.java:748) > > > > Locked ownable synchronizers: > > - <0x000000065cd70a20> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > - <0x000000065ced93f8> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > > > "Reports (Complete)-3" #294 prio=5 os_prio=0 tid=0x00007fb395a6b000 > nid=0x7fbd5 in Object.wait() [0x00007fb3390a4000] > > java.lang.Thread.State: WAITING (on object monitor) > > at java.lang.Object.wait(Native Method) > > at java.lang.Object.wait(Object.java:502) > > at > io.netty.util.concurrent.DefaultPromise.awaitUninterruptibly(DefaultPromise.java:243) > > - locked <0x0000000796e9a460> (a > io.netty.channel.group.DefaultChannelGroupFuture) > > at > io.netty.channel.group.DefaultChannelGroupFuture.awaitUninterruptibly(DefaultChannelGroupFuture.java:183) > > at > io.netty.channel.group.DefaultChannelGroupFuture.awaitUninterruptibly(DefaultChannelGroupFuture.java:40) > > at > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector.close(NettyConnector.java:718) > > - locked <0x000000065d080e98> (a > org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.failoverOrReconnect(ClientSessionFactoryImpl.java:607) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.handleConnectionFailure(ClientSessionFactoryImpl.java:507) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.access$600(ClientSessionFactoryImpl.java:73) > > at > org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingFailureListener.connectionFailed(ClientSessionFactoryImpl.java:1222) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.callFailureListeners(AbstractRemotingConnection.java:75) > > at > org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:219) > > at > org.apache.activemq.artemis.spi.core.protocol.AbstractRemotingConnection.fail(AbstractRemotingConnection.java:218) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:455) > > - locked <0x0000000794a5c028> (a java.lang.Object) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:358) > > at > org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.closeConsumer(ActiveMQSessionContext.java:433) > > at > org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.doCleanUp(ClientConsumerImpl.java:1113) > > at > org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.close(ClientConsumerImpl.java:462) > > at > org.apache.activemq.artemis.jms.client.ActiveMQMessageConsumer.close(ActiveMQMessageConsumer.java:145) > > at > org.springframework.jms.support.JmsUtils.closeMessageConsumer(JmsUtils.java:151) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:356) > > at > org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:235) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1166) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1158) > > at > org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1055) > > at java.lang.Thread.run(Thread.java:748) > > > > Locked ownable synchronizers: > > - <0x0000000794a5bfc8> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > - <0x000000065d0cf390> (a > java.util.concurrent.locks.ReentrantLock$NonfairSync) > > > > > > --- > > > > Now about configuration. > > > > Message sending is done via JmsTemplates. They're configured like this: > > --- > > final JmsTemplate jmsTemplate = new JmsTemplate(connectionFactory); > > jmsTemplate.setSessionTransacted(true); > > jmsTemplate.setSessionAcknowledgeMode(Session.AUTO_ACKNOWLEDGE); > > jmsTemplate.setExplicitQosEnabled(true); > > jmsTemplate.setPriority(Jms.DEFAULT_PRIORITY); > > jmsTemplate.setReceiveTimeout(1000); > > --- > > > > Message receiving is implemented with multiple ListenerContainers. > > One group of containers configured like that: > > --- > > DefaultMessageListenerContainer container = new > BackGroundClusterNodeListenerContainer(args); > > container.setConnectionFactory(connectionFactory); > > container.setCacheLevel(2); > > container.setReceiveTimeout(1500); > > container.setDestinationName(destinationName); > > container.setConcurrency("8-8"); > > container.setBeanName(containerID); > > container.setAutoStartup(true); > > container.setMessageListener(messageListener); > > container.setSessionTransacted(false); > > container.setSessionAcknowledgeMode(AUTO_ACKNOWLEDGE); > > --- > > > > Another one is like that: > > --- > > DefaultMessageListenerContainer container = new > DefaultMessageListenerContainer(); > > container.setDestinationName(destinationName); > > container.setConnectionFactory(connectionFactory); > > container.setConcurrentConsumers(3) > > container.setSessionAcknowledgeMode(101) > > > container.setCacheLevel(modules.reportBuilderHelper.MESSAGE_LISTENER_CACHE_LEVEL) > > container.setExceptionListener(t -> > > { > > Throwable rootCause = ExceptionUtils.getRootCause(t); > > LOG.error("An error occured in listener for " + destination, > rootCause); > > }); > > container.afterPropertiesSet(); > > container.start(); > > --- > > > > Connection factory for the first two cases is created like that: > > --- > > final Map<String, Object> nettyParams = new HashMap<>(2); > > nettyParams.put("host", host); > > nettyParams.put("port", port); > > nettyParams.put(TransportConstants.NETTY_CONNECT_TIMEOUT, 60000); > > nettyParams.put(TransportConstants.REMOTING_THREADS_PROPNAME, 4); > > ActiveMQConnectionFactory connectionFactory = > ActiveMQJMSClient.createConnectionFactoryWithoutHA(JMSFactoryType.CF, > > new TransportConfiguration(NettyConnectorFactory.class.getName(), > nettyParams)) > > > > --- > > > > Connection factory for the last case is created like that (Groovy code): > > --- > > def nettyParams = ["host" : host, "port" : port]; > > def connectionFactory = > ActiveMQJMSClient.createConnectionFactoryWithoutHA(JMSFactoryType.CF, new > TransportConfiguration(NettyConnectorFactory.class.getName(), nettyParams)); > > def serverLocator = connectionFactory.getServerLocator() > > serverLocator.setConnectionTTL(-1) > > serverLocator.setReconnectAttempts(-1) > > serverLocator.setRetryInterval(1000) > > serverLocator.setRetryIntervalMultiplier(1) > > --- > > > > BackGroundClusterNodeListenerContainer - is an extension of a > DefaultMessageListenerContainer with overridden doReceiveAndExecute. The > overridden method does some quick checks and returns false if they don't > hold, otherwise delegates to the super. > > > > Both a client and a broker are on the same laptop. > > > > Any chance there is something I can do to overcome it? > > > > -- > > Oleg Kleshchev -- Clebert Suconic
