[jira] [Created] (FLINK-22883) Select view columns fail when store metadata with hive

[ELLEX_SHEN (Jira)]

ELLEX_SHEN created FLINK-22883:
----------------------------------

             Summary: Select view columns fail when store metadata with hive
                 Key: FLINK-22883
                 URL: https://issues.apache.org/jira/browse/FLINK-22883
             Project: Flink
          Issue Type: Bug
          Components: Table SQL / API
    Affects Versions: 1.13.1
            Reporter: ELLEX_SHEN


1、Flink的元数据使用Hive进行保存，新建视图并保存后，Select时会报错找不到列；

2、在V1.12.3版本上没有问题，但是，升级为V1.13.0 / V1.13.1 后，View的还是会报错无法查询到列的问题。

3、实例代码，以下最后一行Inset报错：

-- ------------------------------------------- 0、新建数据库 
-------------------------------------------
-- 初始化
CREATE CATALOG myhive WITH (
 'type' = 'hive',
 'default-database' = 'test'
 -- 'hive-conf-dir' = '/usr/local/share/flink-1.13.0/conf' -- 
默认读取本地hive-site.xml
);
USE CATALOG myhive;
SET table.sql-dialect=default;
-- ------------------------------------------- 1、KAFKA 源系统数据 
-------------------------------------------
DROP TABLE IF EXISTS e_click;

CREATE TABLE e_click (
 user_id STRING,
 ts STRING,
 log_ts TIMESTAMP(3),
 proctime as PROCTIME(),
 WATERMARK FOR log_ts AS log_ts - INTERVAL '5' SECOND
 -- Define watermark on TIMESTAMP column
)with (
 'connector.type' = 'kafka',
 'connector.version' = 'universal',
 'connector.properties.bootstrap.servers' = 
'master.northking.com:9092,slave1.northking.com:9092,slave2.northking.com:9092',
 'connector.topic' = 'user_clicks',
 'connector.properties.group.id' = 'click',
 'connector.startup-mode' = 'latest-offset',
 'format.type' = 'json'
);

DROP VIEW IF EXISTS e_click_v;
create view e_click_v as select * from e_click;

--
SET table.sql-dialect=hive;

drop table IF EXISTS e_click_hive;
CREATE TABLE e_click_hive (
 user_id STRING,
 ts STRING
) PARTITIONED BY (
 dt STRING
) STORED AS parquet TBLPROPERTIES (
 'sink.partition-commit.trigger'='partition-time',
 'partition.time-extractor.timestamp-pattern'='$dt',
 'sink.partition-commit.policy.kind'='metastore,success-file'
);

-- view -> Hive
SET table.sql-dialect=default;
insert into e_click_hive
 select user_id,ts,DATE_FORMAT(log_ts, 'yyyy-MM-dd')
 from e_click_v;

4、报错信息：

Caused by: org.apache.calcite.runtime.CalciteContextException: From line 2, 
column 12 to line 2, column 18: Column 'user_id' not found in any tableCaused 
by: org.apache.calcite.runtime.CalciteContextException: From line 2, column 12 
to line 2, column 18: Column 'user_id' not found in any table at 
sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at 
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
 at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
 at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at 
org.apache.calcite.runtime.Resources$ExInstWithCause.ex(Resources.java:467) at 
org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:883) at 
org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:868) at 
org.apache.calcite.sql.validate.SqlValidatorImpl.newValidationError(SqlValidatorImpl.java:4861)
 at 
org.apache.calcite.sql.validate.DelegatingScope.fullyQualify(DelegatingScope.java:259)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl$Expander.visit(SqlValidatorImpl.java:5833)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:5982)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:5967)
 at org.apache.calcite.sql.SqlIdentifier.accept(SqlIdentifier.java:320) at 
org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectExpr(SqlValidatorImpl.java:5416)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectItem(SqlValidatorImpl.java:398)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelectList(SqlValidatorImpl.java:4061)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3347)
 at 
org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
 at 
org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:997)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:975)
 at org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:232) at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:952)
 at 
org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:704)

 



--
This message was sent by Atlassian Jira
(v8.3.4#803005)