[ https://issues.apache.org/jira/browse/KYLIN-3969?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Tuo Zhu updated KYLIN-3969: --------------------------- Priority: Blocker (was: Major) > BeelineHiveClient can't parse Hive meta data result set with partitioned table > ------------------------------------------------------------------------------ > > Key: KYLIN-3969 > URL: https://issues.apache.org/jira/browse/KYLIN-3969 > Project: Kylin > Issue Type: Bug > Components: Metadata > Affects Versions: v2.6.1 > Environment: Hive 3.1.0, HiveServer2, Beeline(Along with Ambari > 2.7.1.0) > CentOS Linux release 7.6.1810 > Reporter: Tuo Zhu > Priority: Blocker > > When creating a model, you can add tables without partitions, but can't add > ones with partitions. > Web UI pops:"Oops. cannot get HiveTableMeta". > I tried same table in Hive2 with cli, it works. > log prints: > {code:java} > java.lang.RuntimeException: cannot get HiveTableMeta > at > org.apache.kylin.source.hive.HiveMetadataExplorer.loadTableMetadata(HiveMetadataExplorer.java:68) > at > org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:205) > at > org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:131) > at > org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:661) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:742) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) > at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127) > at > org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter.java:215) > at > org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:116) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.header.HeaderWriterFilter.doFilterInternal(HeaderWriterFilter.java:64) > at > org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:56) > at > org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105) > at > org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331) > at > org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:214) > at > org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:177) > at > org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) > at > org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:209) > at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:244) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) > at > org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) > at > org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198) > at > org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) > at > org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:493) > at > org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140) > at > org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81) > at > org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:650) > at > org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) > at > org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342) > at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:800) > at > org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) > at > org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:806) > at > org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1498) > at > org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at > org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.lang.IllegalArgumentException > at com.google.common.base.Preconditions.checkArgument(Preconditions.java:76) > at > org.apache.kylin.source.hive.BeelineHiveClient.parseResultEntry(BeelineHiveClient.java:185) > at > org.apache.kylin.source.hive.BeelineHiveClient.extractHiveTableMeta(BeelineHiveClient.java:175) > at > org.apache.kylin.source.hive.BeelineHiveClient.getHiveTableMeta(BeelineHiveClient.java:155) > at > org.apache.kylin.source.hive.HiveMetadataExplorer.loadTableMetadata(HiveMetadataExplorer.java:66){code} > > After checking the code, I believe the reason is here in > /src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java: > > {code:java} > // code placeholder > private void parseResultEntry(ResultSet resultSet, HiveTableMetaBuilder > builder) throws SQLException{ > List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = > Lists.newArrayList(); > if ("# Partition Information".equals(resultSet.getString(1).trim())) { > resultSet.next(); > Preconditions.checkArgument("# > col_name".equals(resultSet.getString(1).trim())); > resultSet.next(); > > Preconditions.checkArgument("".equals(resultSet.getString(1).trim())); > while (resultSet.next()) { > if ("".equals(resultSet.getString(1).trim())) { > break; > } > partitionColumns.add(new > HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), > resultSet.getString(2).trim(), > resultSet.getString(3).trim())); > } > builder.setPartitionColumns(partitionColumns); > } > {code} > > The code here expects a blank line in "# Partition Information" section after > "# col_name" line, in "desc formatted tablename" output, which does exist in > Hive 2, CLI mode: > {code:java} > # Partition Information > # col_name data_type comment > pt string > {code} > > But in Hive3, Beeline client, it outputs: > {code:java} > | # Partition Information | NULL > | NULL | > | # col_name | data_type > | comment | > | pt | string > | | > {code} > > The blank line is gone, which makes folloing line fails. > {code:java} > Preconditions.checkArgument("".equals(resultSet.getString(1).trim()));{code} > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)