Just got reports that our CAS server was down. The JVM PID is still alive, the CPU utilization is throttling around 4% and the server is at 50% memory capacity. Looking at the logs just before the first report came in:
2014-06-04 13:08:37,194 INFO [com.github.inspektr.audit.support.Slf4jLoggingAuditTrailManager] - Audit trail record BEGIN ============================================================= WHO: someuser WHAT: ST-129-kFVbjggm7wW6Py1mDVsx-localsso.example.org for http://localhost:9100/myapp/shiro-cas ACTION: SERVICE_TICKET_CREATED APPLICATION: CAS WHEN: Wed Jun 04 13:08:37 EDT 2014 CLIENT IP ADDRESS: 10.10.40.17 SERVER IP ADDRESS: 10.10.41.78 ============================================================= 2014-06-04 13:08:37,196 DEBUG [org.jasig.cas.web.flow.TerminateWebSessionListener] - Error getting service from flow state. java.lang.IllegalStateException: No active FlowSession to access; this FlowExecution has ended at org.springframework.webflow.engine.impl.FlowExecutionImpl.getActiveSession(FlowExecutionImpl.java:191) at org.springframework.webflow.engine.impl.RequestControlContextImpl.getFlowScope(RequestControlContextImpl.java:134) at org.jasig.cas.web.support.WebUtils.getService_aroundBody8(WebUtils.java:87) at org.jasig.cas.web.support.WebUtils.getService_aroundBody9$advice(WebUtils.java:57) at org.jasig.cas.web.support.WebUtils.getService(WebUtils.java:1) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody0(TerminateWebSessionListener.java:62) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody1$advice(TerminateWebSessionListener.java:57) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted(TerminateWebSessionListener.java:1) at org.springframework.webflow.engine.impl.FlowExecutionListeners.fireSessionStarted(FlowExecutionListeners.java:126) at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:367) at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:225) at org.springframework.webflow.executor.FlowExecutorImpl.launchExecution(FlowExecutorImpl.java:140) at org.springframework.webflow.mvc.servlet.FlowHandlerAdapter.handle(FlowHandlerAdapter.java:193) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778) at javax.servlet.http.HttpServlet.service(HttpServlet.java:621) at javax.servlet.http.HttpServlet.service(HttpServlet.java:722) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody2(SafeDispatcherServlet.java:128) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody3$advice(SafeDispatcherServlet.java:57) at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:1) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at com.github.inspektr.common.web.ClientInfoThreadLocalFilter.doFilter(ClientInfoThreadLocalFilter.java:63) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99) at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:929) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1002) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:585) at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 2014-06-04 13:08:37,196 DEBUG [org.jasig.cas.web.flow.TerminateWebSessionListener] - Error getting service from flow state. java.lang.IllegalStateException: No active FlowSession to access; this FlowExecution has ended at org.springframework.webflow.engine.impl.FlowExecutionImpl.getActiveSession(FlowExecutionImpl.java:191) at org.springframework.webflow.engine.impl.RequestControlContextImpl.getFlowScope(RequestControlContextImpl.java:134) at org.jasig.cas.web.support.WebUtils.getService_aroundBody8(WebUtils.java:87) at org.jasig.cas.web.support.WebUtils.getService_aroundBody9$advice(WebUtils.java:57) at org.jasig.cas.web.support.WebUtils.getService(WebUtils.java:1) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody0(TerminateWebSessionListener.java:62) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted_aroundBody1$advice(TerminateWebSessionListener.java:57) at org.jasig.cas.web.flow.TerminateWebSessionListener.sessionStarted(TerminateWebSessionListener.java:1) at org.springframework.webflow.engine.impl.FlowExecutionListeners.fireSessionStarted(FlowExecutionListeners.java:126) at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:367) at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:225) at org.springframework.webflow.executor.FlowExecutorImpl.launchExecution(FlowExecutorImpl.java:140) at org.springframework.webflow.mvc.servlet.FlowHandlerAdapter.handle(FlowHandlerAdapter.java:193) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:923) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778) at javax.servlet.http.HttpServlet.service(HttpServlet.java:621) at javax.servlet.http.HttpServlet.service(HttpServlet.java:722) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody2(SafeDispatcherServlet.java:128) at org.jasig.cas.web.init.SafeDispatcherServlet.service_aroundBody3$advice(SafeDispatcherServlet.java:57) at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:1) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at com.github.inspektr.common.web.ClientInfoThreadLocalFilter.doFilter(ClientInfoThreadLocalFilter.java:63) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99) at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:929) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1002) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:585) at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:310) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Any idea as to what could be going on here? Thanks in advance! -- You are currently subscribed to cas-user@lists.jasig.org as: arch...@mail-archive.com To unsubscribe, change settings or access archives, see http://www.ja-sig.org/wiki/display/JSG/cas-user
