Have you followed the guide: http://kylin.apache.org/docs21/tutorial/cube_spark.html
Please describe the reproduce steps, especially the environment variable exported. 2017-08-23 17:35 GMT+08:00 kylin_liu <allen...@163.com>: > 现在准备用spark 做计算引擎，样本cube 编译的时候就出现问题了，各位是否有碰到这个问题呢？ > > 2017-08-23 04:22:33,551 ERROR [http-bio-7070-exec-9] > controller.CubeController:305 : > org/apache/spark/api/java/function/PairFunction > java.lang.NoClassDefFoundError: > org/apache/spark/api/java/function/PairFunction > at > org.apache.kylin.engine.spark.SparkBatchCubingJobBuilder2. > addLayerCubingSteps(SparkBatchCubingJobBuilder2.java:48) > at > org.apache.kylin.engine.mr.BatchCubingJobBuilder2.build( > BatchCubingJobBuilder2.java:66) > at > org.apache.kylin.engine.spark.SparkBatchCubingEngine2. > createBatchCubingJob(SparkBatchCubingEngine2.java:30) > at > org.apache.kylin.engine.EngineFactory.createBatchCubingJob( > EngineFactory.java:56) > at > org.apache.kylin.rest.service.JobService.submitJobInternal( > JobService.java:238) > at org.apache.kylin.rest.service.JobService.submitJob( > JobService.java:208) > at > org.apache.kylin.rest.service.JobService$$FastClassBySpringCGLIB$$ > 83a44b2a.invoke(<generated>) > at org.springframework.cglib.proxy.MethodProxy.invoke( > MethodProxy.java:204) > at > org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation. > invokeJoinpoint(CglibAopProxy.java:720) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed( > ReflectiveMethodInvocation.java:157) > at > org.springframework.security.access.intercept.aopalliance. > MethodSecurityInterceptor.invoke(MethodSecurityInterceptor.java:68) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed( > ReflectiveMethodInvocation.java:179) > at > org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor. > intercept(CglibAopProxy.java:655) > at > org.apache.kylin.rest.service.JobService$$EnhancerBySpringCGLIB$$ > 39b3ce22.submitJob(<generated>) > at > org.apache.kylin.rest.controller.CubeController. > buildInternal(CubeController.java:302) > at > org.apache.kylin.rest.controller.CubeController. > rebuild(CubeController.java:264) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java: > 62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke( > InvocableHandlerMethod.java:221) > at > org.springframework.web.method.support.InvocableHandlerMethod. > invokeForRequest(InvocableHandlerMethod.java:136) > at > org.springframework.web.servlet.mvc.method.annotation. > ServletInvocableHandlerMethod.invokeAndHandle( > ServletInvocableHandlerMethod.java:110) > at > org.springframework.web.servlet.mvc.method.annotation. > RequestMappingHandlerAdapter.invokeHandlerMethod( > RequestMappingHandlerAdapter.java:832) > at > org.springframework.web.servlet.mvc.method.annotation. > RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter. > java:743) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter. > handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet. > doDispatch(DispatcherServlet.java:961) > at > org.springframework.web.servlet.DispatcherServlet. > doService(DispatcherServlet.java:895) > at > org.springframework.web.servlet.FrameworkServlet.processRequest( > FrameworkServlet.java:967) > at > org.springframework.web.servlet.FrameworkServlet. > doPut(FrameworkServlet.java:880) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:653) > at > org.springframework.web.servlet.FrameworkServlet. > service(FrameworkServlet.java:843) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:303) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at org.apache.tomcat.websocket.server.WsFilter.doFilter( > WsFilter.java:52) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:316) > at > org.springframework.security.web.access.intercept. > FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:126) > at > org.springframework.security.web.access.intercept. > FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:90) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.access.ExceptionTranslationFilter. > doFilter(ExceptionTranslationFilter.java:114) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.session.SessionManagementFilter.doFilter( > SessionManagementFilter.java:122) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication. > AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter. > java:111) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.servletapi.SecurityContextHolderAwareRequ > estFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:169) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.savedrequest.RequestCacheAwareFilter. > doFilter(RequestCacheAwareFilter.java:48) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication.www. > BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter. > java:158) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication. > AbstractAuthenticationProcessingFilter.doFilter( > AbstractAuthenticationProcessingFilter.java:205) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication.logout. > LogoutFilter.doFilter(LogoutFilter.java:120) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.header.HeaderWriterFilter. > doFilterInternal(HeaderWriterFilter.java:64) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.context.request.async. > WebAsyncManagerIntegrationFilter.doFilterInternal( > WebAsyncManagerIntegrationFilter.java:53) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.context.SecurityContextPersistenceFilt > er.doFilter(SecurityContextPersistenceFilter.java:91) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.FilterChainProxy.doFilterInternal( > FilterChainProxy.java:213) > at > org.springframework.security.web.FilterChainProxy.doFilter( > FilterChainProxy.java:176) > at > org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate( > DelegatingFilterProxy.java:346) > at > org.springframework.web.filter.DelegatingFilterProxy.doFilter( > DelegatingFilterProxy.java:262) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at com.thetransactioncompany.cors.CORSFilter.doFilter( > CORSFilter.java:209) > at com.thetransactioncompany.cors.CORSFilter.doFilter( > CORSFilter.java:244) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at > org.apache.catalina.core.StandardWrapperValve.invoke( > StandardWrapperValve.java:220) > at > org.apache.catalina.core.StandardContextValve.invoke( > StandardContextValve.java:122) > at > org.apache.catalina.authenticator.AuthenticatorBase.invoke( > AuthenticatorBase.java:505) > at > org.apache.catalina.core.StandardHostValve.invoke( > StandardHostValve.java:169) > at > org.apache.catalina.valves.ErrorReportValve.invoke( > ErrorReportValve.java:103) > at > org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956) > at > org.apache.catalina.core.StandardEngineValve.invoke( > StandardEngineValve.java:116) > at > org.apache.catalina.connector.CoyoteAdapter.service( > CoyoteAdapter.java:436) > at > org.apache.coyote.http11.AbstractHttp11Processor.process( > AbstractHttp11Processor.java:1078) > at > org.apache.coyote.AbstractProtocol$AbstractConnectionHandler. > process(AbstractProtocol.java:625) > at > org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor. > run(JIoEndpoint.java:318) > at > java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:617) > at > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run( > TaskThread.java:61) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.ClassNotFoundException: > org.apache.spark.api.java.function.PairFunction > at > org.apache.catalina.loader.WebappClassLoaderBase.loadClass( > WebappClassLoaderBase.java:1858) > at > org.apache.catalina.loader.WebappClassLoaderBase.loadClass( > WebappClassLoaderBase.java:1701) > ... 92 more > 2017-08-23 04:22:33,554 ERROR [http-bio-7070-exec-9] > controller.BasicController:57 : > org.apache.kylin.rest.exception.InternalErrorException: > org/apache/spark/api/java/function/PairFunction > at > org.apache.kylin.rest.controller.CubeController. > buildInternal(CubeController.java:306) > at > org.apache.kylin.rest.controller.CubeController. > rebuild(CubeController.java:264) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java: > 62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke( > InvocableHandlerMethod.java:221) > at > org.springframework.web.method.support.InvocableHandlerMethod. > invokeForRequest(InvocableHandlerMethod.java:136) > at > org.springframework.web.servlet.mvc.method.annotation. > ServletInvocableHandlerMethod.invokeAndHandle( > ServletInvocableHandlerMethod.java:110) > at > org.springframework.web.servlet.mvc.method.annotation. > RequestMappingHandlerAdapter.invokeHandlerMethod( > RequestMappingHandlerAdapter.java:832) > at > org.springframework.web.servlet.mvc.method.annotation. > RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter. > java:743) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter. > handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet. > doDispatch(DispatcherServlet.java:961) > at > org.springframework.web.servlet.DispatcherServlet. > doService(DispatcherServlet.java:895) > at > org.springframework.web.servlet.FrameworkServlet.processRequest( > FrameworkServlet.java:967) > at > org.springframework.web.servlet.FrameworkServlet. > doPut(FrameworkServlet.java:880) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:653) > at > org.springframework.web.servlet.FrameworkServlet. > service(FrameworkServlet.java:843) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:303) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at org.apache.tomcat.websocket.server.WsFilter.doFilter( > WsFilter.java:52) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:316) > at > org.springframework.security.web.access.intercept. > FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:126) > at > org.springframework.security.web.access.intercept. > FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:90) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.access.ExceptionTranslationFilter. > doFilter(ExceptionTranslationFilter.java:114) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.session.SessionManagementFilter.doFilter( > SessionManagementFilter.java:122) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication. > AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter. > java:111) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.servletapi.SecurityContextHolderAwareRequ > estFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:169) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.savedrequest.RequestCacheAwareFilter. > doFilter(RequestCacheAwareFilter.java:48) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication.www. > BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter. > java:158) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication. > AbstractAuthenticationProcessingFilter.doFilter( > AbstractAuthenticationProcessingFilter.java:205) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.authentication.logout. > LogoutFilter.doFilter(LogoutFilter.java:120) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.header.HeaderWriterFilter. > doFilterInternal(HeaderWriterFilter.java:64) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.context.request.async. > WebAsyncManagerIntegrationFilter.doFilterInternal( > WebAsyncManagerIntegrationFilter.java:53) > at > org.springframework.web.filter.OncePerRequestFilter. > doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.context.SecurityContextPersistenceFilt > er.doFilter(SecurityContextPersistenceFilter.java:91) > at > org.springframework.security.web.FilterChainProxy$ > VirtualFilterChain.doFilter(FilterChainProxy.java:330) > at > org.springframework.security.web.FilterChainProxy.doFilterInternal( > FilterChainProxy.java:213) > at > org.springframework.security.web.FilterChainProxy.doFilter( > FilterChainProxy.java:176) > at > org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate( > DelegatingFilterProxy.java:346) > at > org.springframework.web.filter.DelegatingFilterProxy.doFilter( > DelegatingFilterProxy.java:262) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at com.thetransactioncompany.cors.CORSFilter.doFilter( > CORSFilter.java:209) > at com.thetransactioncompany.cors.CORSFilter.doFilter( > CORSFilter.java:244) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter( > ApplicationFilterChain.java:241) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter( > ApplicationFilterChain.java:208) > at > org.apache.catalina.core.StandardWrapperValve.invoke( > StandardWrapperValve.java:220) > at > org.apache.catalina.core.StandardContextValve.invoke( > StandardContextValve.java:122) > at > org.apache.catalina.authenticator.AuthenticatorBase.invoke( > AuthenticatorBase.java:505) > at > org.apache.catalina.core.StandardHostValve.invoke( > StandardHostValve.java:169) > at > org.apache.catalina.valves.ErrorReportValve.invoke( > ErrorReportValve.java:103) > at > org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956) > at > org.apache.catalina.core.StandardEngineValve.invoke( > StandardEngineValve.java:116) > at > org.apache.catalina.connector.CoyoteAdapter.service( > CoyoteAdapter.java:436) > at > org.apache.coyote.http11.AbstractHttp11Processor.process( > AbstractHttp11Processor.java:1078) > at > org.apache.coyote.AbstractProtocol$AbstractConnectionHandler. > process(AbstractProtocol.java:625) > at > org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor. > run(JIoEndpoint.java:318) > at > java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:617) > at > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run( > TaskThread.java:61) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.NoClassDefFoundError: > org/apache/spark/api/java/function/PairFunction > at > org.apache.kylin.engine.spark.SparkBatchCubingJobBuilder2. > addLayerCubingSteps(SparkBatchCubingJobBuilder2.java:48) > at > org.apache.kylin.engine.mr.BatchCubingJobBuilder2.build( > BatchCubingJobBuilder2.java:66) > at > org.apache.kylin.engine.spark.SparkBatchCubingEngine2. > createBatchCubingJob(SparkBatchCubingEngine2.java:30) > at > org.apache.kylin.engine.EngineFactory.createBatchCubingJob( > EngineFactory.java:56) > at > org.apache.kylin.rest.service.JobService.submitJobInternal( > JobService.java:238) > at org.apache.kylin.rest.service.JobService.submitJob( > JobService.java:208) > at > org.apache.kylin.rest.service.JobService$$FastClassBySpringCGLIB$$ > 83a44b2a.invoke(<generated>) > at org.springframework.cglib.proxy.MethodProxy.invoke( > MethodProxy.java:204) > at > org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation. > invokeJoinpoint(CglibAopProxy.java:720) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed( > ReflectiveMethodInvocation.java:157) > at > org.springframework.security.access.intercept.aopalliance. > MethodSecurityInterceptor.invoke(MethodSecurityInterceptor.java:68) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed( > ReflectiveMethodInvocation.java:179) > at > org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor. > intercept(CglibAopProxy.java:655) > at > org.apache.kylin.rest.service.JobService$$EnhancerBySpringCGLIB$$ > 39b3ce22.submitJob(<generated>) > at > org.apache.kylin.rest.controller.CubeController. > buildInternal(CubeController.java:302) > ... 77 more > Caused by: java.lang.ClassNotFoundException: > org.apache.spark.api.java.function.PairFunction > at > org.apache.catalina.loader.WebappClassLoaderBase.loadClass( > WebappClassLoaderBase.java:1858) > at > org.apache.catalina.loader.WebappClassLoaderBase.loadClass( > WebappClassLoaderBase.java:1701) > ... 92 more > > -- > View this message in context: http://apache-kylin.74782.x6. > nabble.com/KYLIN2-1-tp8782.html > Sent from the Apache Kylin mailing list archive at Nabble.com. >
