Hi JBMGroup. We got a trouble about JBM.Using Jboss 4.22, JBM 1.4 sp3 cp04.After Long time run and load increase,the JBM Server lose any response. part thread dump:
| "WorkerThread#160[192.166.32.8:33764]" prio=3 tid=0x01664d88 nid=0x4d00 waiting for monitor entry [0x4927f000..0x4927fc70] | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:234) | - waiting to lock <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489) | at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:748) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) | at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) | at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) | at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809) | at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815) | at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) | at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758) | at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:572) | at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387) | at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166) | | "WorkerThread#159[192.166.32.8:33676]" prio=3 tid=0x0242f938 nid=0x4cdd runnable [0x4667e000..0x4667f9f0] | at java.net.SocketOutputStream.socketWrite0(Native Method) | at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) | at java.net.SocketOutputStream.write(SocketOutputStream.java:136) | at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65) | at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123) | - locked <0x7af9d5c0> (a java.io.BufferedOutputStream) | at java.io.DataOutputStream.flush(DataOutputStream.java:106) | at org.jboss.jms.wireformat.ClientDelivery.write(ClientDelivery.java:93) | at org.jboss.jms.wireformat.JMSWireFormat.write(JMSWireFormat.java:237) | at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.versionedWrite(MicroSocketClientInvoker.java:945) | at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:586) | at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:418) | at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:122) | at org.jboss.remoting.Client.invoke(Client.java:1634) | at org.jboss.remoting.Client.invoke(Client.java:548) | at org.jboss.remoting.Client.invokeOneway(Client.java:598) | at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallback(ServerInvokerCallbackHandler.java:826) | at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallbackOneway(ServerInvokerCallbackHandler.java:697) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.performDelivery(ServerSessionEndpoint.java:1432) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.handleDelivery(ServerSessionEndpoint.java:1344) | - locked <0x794462d8> (a org.jboss.jms.server.endpoint.ServerSessionEndpoint) | at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:322) | - locked <0x7a778760> (a java.lang.Object) | at org.jboss.messaging.core.impl.RoundRobinDistributor.handle(RoundRobinDistributor.java:119) | at org.jboss.messaging.core.impl.MessagingQueue$DistributorWrapper.handle(MessagingQueue.java:582) | at org.jboss.messaging.core.impl.ClusterRoundRobinDistributor.handle(ClusterRoundRobinDistributor.java:79) | at org.jboss.messaging.core.impl.ChannelSupport.deliverInternal(ChannelSupport.java:606) | at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:505) | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:238) | - locked <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489) | at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:748) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) | at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) | at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) | at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809) | at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815) | at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) | at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758) | at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:572) | at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387) | at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166) | | "WorkerThread#158[192.166.32.8:33596]" prio=3 tid=0x010b6770 nid=0x4c90 waiting for monitor entry [0x4f67f000..0x4f67fc70] | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:234) | - waiting to lock <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489) | at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:748) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) | at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) | at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) | at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809) | at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815) | at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) | at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758) | at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:572) | at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387) | at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166) | | "WorkerThread#157[192.166.32.8:33731]" prio=3 tid=0x01c23f78 nid=0x4c78 waiting for monitor entry [0x4967f000..0x4967fc70] | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:234) | - waiting to lock <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489) | at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:748) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) | at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) | at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) | at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809) | at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815) | at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) | at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758) | at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:572) | at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:373) | at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166) | | "WorkerThread#53[192.166.32.8:33606]" prio=3 tid=0x02497750 nid=0x281 waiting for monitor entry [0x5757f000..0x5757fbf0] | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:234) | - waiting to lock <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:489) | at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:748) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:383) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) | at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) | at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) | at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) | at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) | at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:809) | at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1815) | at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) | at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1826) | at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:758) | at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:572) | at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387) | at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166) | | "Thread-117" prio=3 tid=0x00f0b818 nid=0x21c waiting for monitor entry [0x4fe7f000..0x4fe7fa70] | at org.jboss.messaging.core.impl.ChannelSupport.deliver(ChannelSupport.java:350) | - waiting to lock <0x796c3d18> (a java.lang.Object) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint$2.run(ServerSessionEndpoint.java:1539) | at EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:89) | at java.lang.Thread.run(Thread.java:595) All Thread are waiting to lock 0x796c3d18,but the lock holder are waiting to socket write. "WorkerThread#159[192.166.32.8:33676]" prio=3 tid=0x0242f938 nid=0x4cdd runnable [0x4667e000..0x4667f9f0] | at java.net.SocketOutputStream.socketWrite0(Native Method) | at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) | at java.net.SocketOutputStream.write(SocketOutputStream.java:136) | at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65) | at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123) | - locked <0x7af9d5c0> (a java.io.BufferedOutputStream) | at java.io.DataOutputStream.flush(DataOutputStream.java:106) | at org.jboss.jms.wireformat.ClientDelivery.write(ClientDelivery.java:93) | at org.jboss.jms.wireformat.JMSWireFormat.write(JMSWireFormat.java:237) | at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.versionedWrite(MicroSocketClientInvoker.java:945) | at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:586) | at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:418) | at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:122) | at org.jboss.remoting.Client.invoke(Client.java:1634) | at org.jboss.remoting.Client.invoke(Client.java:548) | at org.jboss.remoting.Client.invokeOneway(Client.java:598) | at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallback(ServerInvokerCallbackHandler.java:826) | at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallbackOneway(ServerInvokerCallbackHandler.java:697) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.performDelivery(ServerSessionEndpoint.java:1432) | at org.jboss.jms.server.endpoint.ServerSessionEndpoint.handleDelivery(ServerSessionEndpoint.java:1344) | - locked <0x794462d8> (a org.jboss.jms.server.endpoint.ServerSessionEndpoint) | at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:322) | - locked <0x7a778760> (a java.lang.Object) | at org.jboss.messaging.core.impl.RoundRobinDistributor.handle(RoundRobinDistributor.java:119) | at org.jboss.messaging.core.impl.MessagingQueue$DistributorWrapper.handle(MessagingQueue.java:582) | at org.jboss.messaging.core.impl.ClusterRoundRobinDistributor.handle(ClusterRoundRobinDistributor.java:79) | at org.jboss.messaging.core.impl.ChannelSupport.deliverInternal(ChannelSupport.java:606) | at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:505) | at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:238) | - locked <0x796c3d18> (a java.lang.Object) | at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2203) the socket write thread infinite writing,can not return. we use netstat,found the port 4557 tcp recieve window is zero. 192.166.32.8.4557 192.166.32.21.2154 65535 0 0 0 ESTABLISHED 192.166.32.8.4557 192.166.32.21.2152 65535 0 0 0 ESTABLISHED 192.166.32.8.33676 192.166.32.8.4557 49152 0 49152 0 ESTABLISHED 192.166.32.8.4557 192.166.32.8.33676 49152 0 49152 0 ESTABLISHED the port 33676's receive window seems normal,but due to local network address,the output of loopback info can be ignore(The answer of Sun Consult). The things are seem to be: the lock holder thread are writing to port 4557,but the receive window is zero,can not write. the other reader thread writing to a lock to read from 4557. Is the probem biSocket ? BioSocket, View the original post : http://www.jboss.com/index.html?module=bb&op=viewtopic&p=4191447#4191447 Reply to the post : http://www.jboss.com/index.html?module=bb&op=posting&mode=reply&p=4191447 _______________________________________________ jboss-user mailing list jboss-user@lists.jboss.org https://lists.jboss.org/mailman/listinfo/jboss-user
