[
https://issues.apache.org/jira/browse/KYLIN-4396?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
nichunen updated KYLIN-4396:
----------------------------
Fix Version/s: (was: v2.6.6)
> File Descriptor Leakage in MR Build Engine
> ------------------------------------------
>
> Key: KYLIN-4396
> URL: https://issues.apache.org/jira/browse/KYLIN-4396
> Project: Kylin
> Issue Type: Bug
> Components: Tools, Build and Test
> Affects Versions: v3.0.0, v3.0.1
> Reporter: Xiaoxiang Yu
> Assignee: Xiaoxiang Yu
> Priority: Major
> Fix For: v3.1.0, v3.0.2
>
> Attachments: image-2020-03-03-11-26-17-536.png,
> image-2020-03-03-11-26-47-382.png, image-2020-03-03-11-27-35-009.png,
> image-2020-03-03-11-28-31-161.png, image-2020-03-03-11-47-15-845.png,
> image-2020-03-04-09-24-32-013.png
>
>
> After runing for one or two days, Job Server failed to response to any http
> requests, but the process looks good, without high CPU pressure etc. In
> _kylin.out_, we find error message indicate that process has hit the limit of
> *FD.*
> After checking detail, we find many fds are not released after a long period,
> most of them are socket in state, that is *CLOSE_WAIT*.
> !https://user-images.githubusercontent.com/14030549/72678560-0fc7e580-3ae2-11ea-85ea-5dcb40e97bd7.png!
>
> !https://user-images.githubusercontent.com/14030549/72678638-c9bf5180-3ae2-11ea-880c-7f8e34d17a8c.png!
>
>
> {code:java}
> 一月 19, 2020 5:13:51 下午 org.apache.catalina.core.StandardWrapperValve invoke
> 严重: Servlet.service() for servlet [default] in context with path [/kylin]
> threw exception java.io.FileNotFoundException:
> /root/xiaoxiang.yu/deploy/kylin-all/tomcat/webapps/kylin/js/scripts.min.20191209061232.js
> (打开的文件过多) at java.io.FileInputStream.open0(Native Method) at
> java.io.FileInputStream.open(FileInputStream.java:195) at
> java.io.FileInputStream.<init>(FileInputStream.java:138) at
> org.apache.naming.resources.FileDirContext$FileResource.streamContent(FileDirContext.java:1008)
> at
> org.apache.catalina.servlets.DefaultServlet.copy(DefaultServlet.java:2084) at
> org.apache.catalina.servlets.DefaultServlet.serveResource(DefaultServlet.java:1023)
> at
> org.apache.catalina.servlets.DefaultServlet.doGet(DefaultServlet.java:453) at
> javax.servlet.http.HttpServlet.service(HttpServlet.java:624) at
> org.apache.catalina.servlets.DefaultServlet.service(DefaultServlet.java:433)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)
> at
> org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)
> at
> org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter.java:158)
> at
> org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:116)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.header.HeaderWriterFilter.doFilterInternal(HeaderWriterFilter.java:64)
> at
> org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:56)
> at
> org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105)
> at
> org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
> at
> org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:214)
> at
> org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:177)
> at
> org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
> at
> org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:209)
> at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:244) at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
> at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
> at
> org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137)
> at
> org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
> at
> org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at
> org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
> at java.lang.Thread.run(Thread.java:748)
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
