We recently upgraded to SDK 3.5a (LCDS 2.6.1) and started to see sporadic
deadlocks in LCDS's rtmp thread. These are difficult to reproduce in test
environment but happens in production once every couple of days, sometimes even
more frequently.
Here is the thread dump of the threads in lock.
Found one Java-level deadlock
"my-rtmp-SocketServer-WorkerThread-300":
waiting to lock monitor 0x6a949ba4 (object 0x0b27f6e0, a
flex.data.SequenceManager),
which is held by "my-rtmp-SocketServer-Reactor4Writer"
"my-rtmp-SocketServer-Reactor4Writer":
waiting to lock monitor 0x6a90f2d4 (object 0x0b321d90, a java.util.ArrayList),
which is held by "my-rtmp-SocketServer-Reactor2Writer"
"my-rtmp-SocketServer-Reactor2Writer":
waiting to lock monitor 0x6a6304a4 (object 0x0b27fc88, a java.util.HashMap),
which is held by "my-rtmp-SocketServer-Reactor4Writer"
Java stack information for the threads listed above
"my-rtmp-SocketServer-WorkerThread-300":
at flex.data.SequenceManager.createSequence(SequenceManager.java:2245)
- waiting to lock <0x0b27f6e0> (a flex.data.SequenceManager)
at flex.data.SequenceManager.manageSequence(SequenceManager.java:786)
at flex.data.SequenceManager.manageSequence(SequenceManager.java:755)
at flex.data.DataService.serviceMessage(DataService.java:571)
at
flex.messaging.MessageBroker.routeMessageToService(MessageBroker.java:1495)
at
flex.messaging.endpoints.AbstractEndpoint.serviceMessage(AbstractEndpoint.java:882)
at
flex.messaging.endpoints.RTMPConnection.handleTCCommand(RTMPConnection.java:722)
at
flex.messaging.endpoints.RTMPConnection.serviceTCMessage(RTMPConnection.java:1021)
at
flex.messaging.endpoints.RTMPConnection.doRead(RTMPConnection.java:514)
at
flex.messaging.endpoints.RTMPProtocolHandler.doRead(RTMPProtocolHandler.java:123)
at
flex.messaging.socketserver.Connection$ConnectionReader.run(Connection.java:778)
at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1061)
at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:575)
at java.lang.Thread.run(Thread.java:619)
"my-rtmp-SocketServer-Reactor4Writer":
at flex.data.SequenceManager.getSequence(SequenceManager.java:373)
- waiting to lock <0x0b321d90> (a java.util.ArrayList)
at
flex.data.SequenceManager.removeItemIfNecessary(SequenceManager.java:3652)
at
flex.data.SequenceManager.removeItemIdentityFromSequence(SequenceManager.java:3108)
- locked <0x0b27fc88> (a java.util.HashMap)
at
flex.data.SequenceManager.unsafeRemoveItemIdentitiesFromSequence(SequenceManager.java:747)
at flex.data.SequenceManager.removeSequence(SequenceManager.java:639)
at flex.data.SequenceManager.removeSequence(SequenceManager.java:621)
at flex.data.SequenceManager.unsubscribeClient(SequenceManager.java:548)
- locked <0x0b27fc70> (a java.util.ArrayList)
- locked <0x0b27f6e0> (a flex.data.SequenceManager)
at
flex.data.SequenceManager.messageClientDestroyed(SequenceManager.java:3950)
at flex.messaging.MessageClient.invalidate(MessageClient.java:764)
at flex.messaging.MessageClient.invalidate(MessageClient.java:716)
at flex.messaging.client.FlexClient.invalidate(FlexClient.java:480)
at
flex.messaging.client.FlexClient.unregisterFlexSession(FlexClient.java:1286)
at flex.messaging.FlexSession.unregisterFlexClient(FlexSession.java:850)
at flex.messaging.FlexSession.invalidate(FlexSession.java:666)
at
flex.messaging.endpoints.RTMPProtocolHandler$DirectRTMPConnection.close(RTMPProtocolHandler.java:306)
at
flex.messaging.endpoints.RTMPConnection.close(RTMPConnection.java:212)
at
flex.messaging.endpoints.RTMPProtocolHandler.closed(RTMPProtocolHandler.java:100)
at
flex.messaging.socketserver.Connection.closeImmediate(Connection.java:422)
at
flex.messaging.socketserver.Connection$ConnectionWriter.run(Connection.java:965)
at flex.messaging.socketserver.Reactor$Writer.run(Reactor.java:675)
at java.lang.Thread.run(Thread.java:619)
"my-rtmp-SocketServer-Reactor2Writer":
at
flex.data.SequenceManager.isItemReferencedInternal(SequenceManager.java:3555)
- waiting to lock <0x0b27fc88> (a java.util.HashMap)
at flex.data.SequenceManager.isItemReferenced(SequenceManager.java:3540)
at
flex.data.SequenceManager.isItemReferencedInternal(SequenceManager.java:3567)
- locked <0x0b321da8> (a java.util.HashMap)
at flex.data.SequenceManager.isItemReferenced(SequenceManager.java:3540)
at
flex.data.SequenceManager.isItemReferencedInternal(SequenceManager.java:3567)
- locked <0x0b27bd10> (a java.util.HashMap)
at flex.data.SequenceManager.isItemReferenced(SequenceManager.java:3540)
at
flex.data.SequenceManager.removeItemIfNecessary(SequenceManager.java:3630)
at
flex.data.SequenceManager.removeItemIdentityFromSequence(SequenceManager.java:3108)
- locked <0x0b27bd10> (a java.util.HashMap)
at
flex.data.SequenceManager.unsafeRemoveItemIdentitiesFromSequence(SequenceManager.java:747)
at flex.data.SequenceManager.removeSequence(SequenceManager.java:639)
at
flex.data.SequenceManager.removeItemIfNecessary(SequenceManager.java:3654)
at
flex.data.SequenceManager.removeItemIdentityFromSequence(SequenceManager.java:3108)
- locked <0x0b321da8> (a java.util.HashMap)
at
flex.data.SequenceManager.unsafeRemoveItemIdentitiesFromSequence(SequenceManager.java:747)
at flex.data.SequenceManager.removeSequence(SequenceManager.java:639)
at flex.data.SequenceManager.removeSequence(SequenceManager.java:621)
at flex.data.SequenceManager.unsubscribeClient(SequenceManager.java:548)
- locked <0x0b321d90> (a java.util.ArrayList)
- locked <0x0b3217b0> (a flex.data.SequenceManager)
at
flex.data.SequenceManager.messageClientDestroyed(SequenceManager.java:3950)
at flex.messaging.MessageClient.invalidate(MessageClient.java:764)
at flex.messaging.MessageClient.invalidate(MessageClient.java:716)
at flex.messaging.client.FlexClient.invalidate(FlexClient.java:480)
at
flex.messaging.client.FlexClient.unregisterFlexSession(FlexClient.java:1286)
at flex.messaging.FlexSession.unregisterFlexClient(FlexSession.java:850)
at flex.messaging.FlexSession.invalidate(FlexSession.java:666)
at
flex.messaging.endpoints.RTMPProtocolHandler$DirectRTMPConnection.close(RTMPProtocolHandler.java:306)
at
flex.messaging.endpoints.RTMPConnection.close(RTMPConnection.java:212)
at
flex.messaging.endpoints.RTMPProtocolHandler.closed(RTMPProtocolHandler.java:100)
at
flex.messaging.socketserver.Connection.closeImmediate(Connection.java:422)
at
flex.messaging.socketserver.Connection$ConnectionWriter.run(Connection.java:965)
at flex.messaging.socketserver.Reactor$Writer.run(Reactor.java:675)
at java.lang.Thread.run(Thread.java:619)
-Shailesh
