Hi all,
we seem to have a deadlock situation when a broker shutdown happens.
We're in the middle of onMessage callback and trying to call commit.
The ActiveMQConsumer::dispatch method holds the "unconsumedMessages"
lock and waits for the "reconnectMutex" when committing. The failover
transport calls clearMessagesInProgress which tries to get hold of the
"unconsumedMessages" lock but holds the "reconnectMutex" in
FailoverTransport::handleTransportFailure. The two stack traces below
should show what happens.
We're using ActiveMQ-CPP 3.2.5 with a 5.3.2 broker.
Kind regards,
Chris
One thread here:
ntdll.dll!77dbf8c1()
[Frames below may be incorrect and/or missing, no symbols loaded
for
ntdll.dll]
ntdll.dll!77dbf8c1()
ntdll.dll!77dd8dd4()
kernel32.dll!775c14dd()
>
activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
* handle=0x039e0ed0) Line 68 + 0xa bytes C++
activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
0xe bytes C++
activemq-cppd.dll!decaf::util::StlQueue<decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
>::lock() Line 254 C++
activemq-cppd.dll!activemq::core::MessageDispatchChannel::lock()
Line 154 C++
activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
C++
activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
* object=0x039d4c48, const bool intiallyLocked=true) Line 34 C++
activemq-cppd.dll!activemq::core::ActiveMQConsumer::clearMessagesInProgress()
Line 1112 + 0x17 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQSession::clearMessagesInProgress()
Line 240 C++
activemq-cppd.dll!activemq::core::ActiveMQConnection::transportInterrupted()
Line 705 C++
activemq-cppd.dll!activemq::transport::TransportFilter::transportInterrupted()
Line 67 + 0x13 bytes C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransport::handleTransportFailure(const
decaf::lang::Exception & error={...}) Line 479 C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransportListener::onException(const
decaf::lang::Exception & ex={...}) Line 97 + 0xf bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::inactivity::InactivityMonitor::onException(const
decaf::lang::Exception & ex={...}) Line 276 C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::IOTransport::fire(decaf::lang::Exception
& ex={...}) Line 73 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::IOTransport::run() Line 247
C++
activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
* properties=0x039cb968) Line 135 + 0x11 bytes C++
activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
arg=0x039cb968) Line 208 + 0x9 bytes C++
msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
msvcr100d.dll!_threadstartex(void * ptd=0x0396e030) Line 297 C
kernel32.dll!775c33ca()
ntdll.dll!77dd9ed2()
ntdll.dll!77dd9ea5()
Dispatch thread here:
ntdll.dll!77dbf8c1()
[Frames below may be incorrect and/or missing, no symbols loaded
for
ntdll.dll]
ntdll.dll!77dbf8c1()
ntdll.dll!77dd8dd4()
ntdll.dll!77dd8cb8()
>
activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
* handle=0x039b6988) Line 68 + 0xa bytes C++
activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
0xe bytes C++
activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
C++
activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
* object=0x039d4974, const bool intiallyLocked=true) Line 34 C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransport::oneway(const
decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
& command={...}) Line 186 + 0x20 bytes C++
activemq-cppd.dll!activemq::transport::correlator::ResponseCorrelator::oneway(const
decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
& command={...}) Line 86 C++
activemq-cppd.dll!activemq::core::ActiveMQConnection::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
command={...}) Line 742 C++
activemq-cppd.dll!activemq::core::ActiveMQSession::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
command={...}) Line 904 C++
activemq-cppd.dll!activemq::core::ActiveMQConsumer::acknowledge()
Line 861 C++
activemq-cppd.dll!activemq::core::TransactionSynhcronization::beforeEnd()
Line 86 C++
activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::beforeEnd()
Line 192 + 0x41 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::commit()
Line 130 C++
activemq-cppd.dll!activemq::core::ActiveMQSession::commit() Line
190 C++
<<<<<onMessage call removed>>>>>>>
activemq-cppd.dll!activemq::core::ActiveMQConsumer::dispatch(const
decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
& dispatch={...}) Line 1021 C++
activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::dispatch(const
decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
& dispatch={...}) Line 129 + 0x15 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::iterate()
Line 167 C++
activemq-cppd.dll!activemq::threads::DedicatedTaskRunner::run()
Line 111 + 0x19 bytes C++
activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
* properties=0x039cbd58) Line 135 + 0x11 bytes C++
activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
arg=0x039cbd58) Line 208 + 0x9 bytes C++
msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
msvcr100d.dll!_threadstartex(void * ptd=0x0396dba0) Line 297 C
kernel32.dll!775c33ca()
ntdll.dll!77dd9ed2()
ntdll.dll!77dd9ea5()
