Hi: I am getting this error when we run our application in clustering mode. Initially after restart everything seems to be fine for 15 mins but after sometime we are seeing below exception and then consistently.
| 09:17:25,899 WARN [arjLoggerI18N] [com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator_2] TwoPhaseCoordinator.beforeCompletion - failed for com.arjuna.ats.internal.jta.resources.arjunacore.synchronizationim...@1290b68 | java.lang.RuntimeException: Unable to broadcast invalidation messages | at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:118) | at org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) | at org.jboss.cache.interceptors.TxInterceptor.runPreparePhase(TxInterceptor.java:873) | at org.jboss.cache.interceptors.TxInterceptor$LocalSynchronizationHandler.beforeCompletion(TxInterceptor.java:1138) | at org.jboss.cache.interceptors.OrderedSynchronizationHandler.beforeCompletion(OrderedSynchronizationHandler.java:75) | at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:114) | at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:247) | at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:86) | at com.arjuna.ats.arjuna.AtomicAction.end(AtomicAction.java:216) | at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commit(TransactionImple.java:238) | at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501) | at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361) | at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) | at org.jboss.ejb.plugins.AbstractInterceptor.invoke(AbstractInterceptor.java:111) | at com.appln.foundation.appserver.jboss4.IsolationBoundaryIntercepter.invoke(IsolationBoundaryIntercepter.java:57) | at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168) | at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) | at org.jboss.ejb.plugins.CleanShutdownInterceptor.invoke(CleanShutdownInterceptor.java:278) | at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) | at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648) | at org.jboss.ejb.Container.invoke(Container.java:960) | at sun.reflect.GeneratedMethodAccessor468.invoke(Unknown Source) | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | at java.lang.reflect.Method.invoke(Method.java:585) | at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) | at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) | at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) | at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) | at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) | at org.jboss.invocation.local.LocalInvoker$MBeanServerAction.invoke(LocalInvoker.java:169) | at org.jboss.invocation.local.LocalInvoker.invoke(LocalInvoker.java:118) | at org.jboss.invocation.InvokerInterceptor.invokeLocal(InvokerInterceptor.java:209) | at org.jboss.invocation.InvokerInterceptor.invoke(InvokerInterceptor.java:195) | at org.jboss.proxy.ejb.RetryInterceptor.invoke(RetryInterceptor.java:176) | at org.jboss.proxy.TransactionInterceptor.invoke(TransactionInterceptor.java:61) | at org.jboss.proxy.SecurityInterceptor.invoke(SecurityInterceptor.java:70) | at org.jboss.proxy.ejb.StatelessSessionInterceptor.invoke(StatelessSessionInterceptor.java:112) | at org.jboss.proxy.ClientContainer.invoke(ClientContainer.java:100) | at $Proxy262.getAvailableSearchKeys(Unknown Source) | at com.appln.foundation.webapp.common.tags.QueryBuilderTag.doStartTag(QueryBuilderTag.java:891) | at org.apache.jsp.jsp.hierarchymanagement.advancedSearch_jsp._jspService(advancedSearch_jsp.java:317) | at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) | at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) | at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) | at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590) | at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505) | at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968) | at org.apache.jsp.jsp.cm.hierarchymanagement.abstractUnionListOutput_jsp._jspService(abstractUnionListOutput_jsp.java:672) | at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) | at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) | at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) | at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590) | at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505) | at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968) | at org.apache.jasper.runtime.PageContextImpl.include(PageContextImpl.java:614) | at org.apache.struts.tiles.TilesUtilImpl.doInclude(TilesUtilImpl.java:99) | at org.apache.struts.tiles.TilesUtil.doInclude(TilesUtil.java:135) | at org.apache.struts.taglib.tiles.InsertTag.doInclude(InsertTag.java:760) | at org.apache.struts.taglib.tiles.InsertTag$InsertHandler.doEndTag(InsertTag.java:892) | at org.apache.struts.taglib.tiles.InsertTag.doEndTag(InsertTag.java:462) | at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspx_meth_tiles_005finsert_005f0(emptyAjaxLayout_jsp.java:382) | at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspService(emptyAjaxLayout_jsp.java:167) | at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) | at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) | at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) | at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469) | at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403) | at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301) | at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063) | at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263) | at org.apache.struts.tiles.TilesRequestProcessor.processTilesDefinition(TilesRequestProcessor.java:239) | at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:302) | at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229) | at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133) | at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176) | at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194) | at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) | at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469) | at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403) | at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301) | at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063) | at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263) | at org.apache.struts.action.RequestProcessor.processForwardConfig(RequestProcessor.java:386) | at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:318) | at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229) | at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133) | at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176) | at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194) | at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at com.appln.cm.webapp.common.menu.CmPermissionsFilter.doFilter(CmPermissionsFilter.java:33) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at com.appln.foundation.webapp.authentication.AuthenticationFilterBase.doFilter(AuthenticationFilterBase.java:108) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at com.appln.foundation.webapp.common.compression.GZIPFilter.doFilter(GZIPFilter.java:149) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230) | at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) | at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:179) | at org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:87) | at org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:84) | at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84) | at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128) | at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104) | at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157) | at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) | at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:241) | at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:437) | at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:381) | at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) | at java.lang.Thread.run(Thread.java:595) | Caused by: org.jboss.cache.ReplicationException: rsp=sender=10.196.15.176:55801, retval=null, received=false, suspected=false | at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4422) | at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4344) | at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4455) | at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:110) | at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:88) | at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateAcrossCluster(InvalidationInterceptor.java:160) | at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateModifications(InvalidationInterceptor.java:170) | at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:102) | ... 142 more | Caused by: org.jboss.cache.lock.TimeoutException: Response timed out: sender=10.196.15.176:55801, retval=null, received=false, suspected=false | at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4420) | ... 149 more | | | Any suggestions will be helpful. View the original post : http://www.jboss.com/index.html?module=bb&op=viewtopic&p=4200833#4200833 Reply to the post : http://www.jboss.com/index.html?module=bb&op=posting&mode=reply&p=4200833 _______________________________________________ jboss-user mailing list jboss-user@lists.jboss.org https://lists.jboss.org/mailman/listinfo/jboss-user