This is an automated email from the ASF dual-hosted git repository. difin pushed a commit to branch iceberg_native_views in repository https://gitbox.apache.org/repos/asf/hive.git
commit fcde0f3304ec4c32162e60d532afef2f8a99ba2b Author: Dmitriy Fingerman <[email protected]> AuthorDate: Mon Apr 27 14:20:47 2026 -0400 fixes --- .../org/apache/iceberg/hive/MetastoreUtil.java | 8 + .../test/queries/positive/iceberg_native_view.q | 34 +++- .../positive/iceberg_rest_catalog_gravitino.q | 55 ++++-- .../queries/positive/iceberg_rest_catalog_hms.q | 59 +++++-- .../positive/llap/iceberg_native_view.q.out | 190 ++++++++++++++++++--- .../llap/iceberg_rest_catalog_gravitino.q.out | 8 +- .../positive/llap/iceberg_rest_catalog_hms.q.out | 8 +- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 13 +- .../hadoop/hive/ql/parse/TestParseDefault.java | 4 +- .../ql/ddl/view/create/CreateViewAnalyzer.java | 60 +++---- 10 files changed, 340 insertions(+), 99 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 6ba49b7337d..d3b64496a5c 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -184,6 +184,14 @@ public static Table toHiveView(View view, Configuration conf) { boolean hiveEngineEnabled = false; result.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + // HMS / Hive DESC FORMATTED expect non-null bucket and sort column lists on StorageDescriptor. + StorageDescriptor sd = result.getSd(); + if (sd.getBucketCols() == null) { + sd.setBucketCols(Lists.newArrayList()); + } + if (sd.getSortCols() == null) { + sd.setSortCols(Lists.newArrayList()); + } long maxHiveTablePropertySize = conf.getLong( diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q index 4c954f786f0..eea5657baae 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q @@ -1,4 +1,6 @@ -- SORT_QUERY_RESULTS +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ create database ice_native_view_db; use ice_native_view_db; @@ -28,11 +30,33 @@ update src_ice set last_name = 'ln7a' where first_name='fn7'; delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a'); -create view v_ice as select * from src_ice stored by iceberg; +---------------------------------------------------------------- +-- Explicit native view metadata (STORED BY before AS) +---------------------------------------------------------------- +create view v_ice stored by iceberg as select * from src_ice; select * from v_ice; - +desc formatted v_ice; drop view v_ice; -drop table src_ice; -use default; -drop database ice_native_view_db cascade; + +---------------------------------------------------------------- +-- Regular Hive view from when STORED BY is HiveIcebergStorageHandler +---------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view v_def as select * from src_ice; +select * from v_def; +desc formatted v_def; +drop view v_def; + +---------------------------------------------------------------- +-- Native view from default storage handler when STORED BY is omitted +---------------------------------------------------------------- + +set hive.default.storage.handler.class=; + +create view v_hive as select * from src_ice; +select * from v_hive; +desc formatted v_hive; +drop view v_hive; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 6b73ff2aa13..24d9b2eae49 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -72,21 +72,54 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; ------------------------------------------------------------------------------ ---! Iceberg native views (STORED BY ICEBERG) on REST catalog ------------------------------------------------------------------------------ +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- -- Enable once CBO supports Iceberg native views on partitioned tables set hive.cbo.enable=false; -create view ice_v1 as select * from ice_orc1 stored by iceberg; -create view ice_v2 as select * from ice_orc2 stored by iceberg; - -select * from ice_v1; -select * from ice_v2; - -drop view ice_v1; -drop view ice_v2; +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc1; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc2; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v stored by iceberg as select * from ice_orc1; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc2; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; ----------------------------------------------------------------------------- diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index eefd5bd5ce9..5a4ab05dbae 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -67,21 +67,54 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; ------------------------------------------------------------------------------ ---! Iceberg native views (STORED BY ICEBERG) on REST catalog ------------------------------------------------------------------------------ +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- -- Enable once CBO supports Iceberg native views on partitioned tables -set hive.cbo.enable=false; - -create view ice_v1 as select * from ice_orc1 stored by iceberg; -create view ice_v2 as select * from ice_orc2 stored by iceberg; - -select * from ice_v1; -select * from ice_v2; - -drop view ice_v1; -drop view ice_v2; +set hive.cbo.enable=false; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc1; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc2; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v stored by iceberg as select * from ice_orc1; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v stored by iceberg as select * from ice_orc2; +select * from ice_v; +desc formatted ice_v; +drop view ice_v; ----------------------------------------------------------------------------- diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out index ae1a55ef129..449f9f39f66 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out @@ -162,12 +162,12 @@ POSTHOOK: query: delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a') POSTHOOK: type: QUERY POSTHOOK: Input: ice_native_view_db@src_ice #### A masked pattern was here #### -PREHOOK: query: create view v_ice as select * from src_ice stored by iceberg +PREHOOK: query: create view v_ice stored by iceberg as select * from src_ice PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_native_view_db@src_ice PREHOOK: Output: database:ice_native_view_db PREHOOK: Output: ice_native_view_db@v_ice -POSTHOOK: query: create view v_ice as select * from src_ice stored by iceberg +POSTHOOK: query: create view v_ice stored by iceberg as select * from src_ice POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_native_view_db@src_ice POSTHOOK: Output: database:ice_native_view_db @@ -189,6 +189,42 @@ fn3 ln3a 1 fn4 ln4a 1 fn5 ln5a 2 fn6 ln6a 2 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` PREHOOK: query: drop view v_ice PREHOOK: type: DROPVIEW PREHOOK: Input: ice_native_view_db@v_ice @@ -197,27 +233,141 @@ POSTHOOK: query: drop view v_ice POSTHOOK: type: DROPVIEW POSTHOOK: Input: ice_native_view_db@v_ice POSTHOOK: Output: ice_native_view_db@v_ice -PREHOOK: query: drop table src_ice -PREHOOK: type: DROPTABLE +PREHOOK: query: create view v_def as select * from src_ice +PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_native_view_db@src_ice PREHOOK: Output: database:ice_native_view_db -PREHOOK: Output: ice_native_view_db@src_ice -POSTHOOK: query: drop table src_ice -POSTHOOK: type: DROPTABLE +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view v_def as select * from src_ice +POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_native_view_db@src_ice POSTHOOK: Output: database:ice_native_view_db -POSTHOOK: Output: ice_native_view_db@src_ice -PREHOOK: query: use default -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:default -POSTHOOK: query: use default -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:default -PREHOOK: query: drop database ice_native_view_db cascade -PREHOOK: type: DROPDATABASE -PREHOOK: Input: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 +PREHOOK: query: desc formatted v_def +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_def +POSTHOOK: query: desc formatted v_def +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_def +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: create view v_hive as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice PREHOOK: Output: database:ice_native_view_db -POSTHOOK: query: drop database ice_native_view_db cascade -POSTHOOK: type: DROPDATABASE -POSTHOOK: Input: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: create view v_hive as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: Lineage: v_hive.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_hive.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_hive.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_hive +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +POSTHOOK: query: select * from v_hive +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 +PREHOOK: query: desc formatted v_hive +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: query: desc formatted v_hive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_hive +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select * from src_ice +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_hive +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: drop view v_hive +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: ice_native_view_db@v_hive diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 0c27b278031..e76bf23e009 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -180,12 +180,12 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc1 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc1 POSTHOOK: Output: database:ice_rest @@ -195,12 +195,12 @@ POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:de POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 3a046d00a7f..b42301b58dd 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -180,12 +180,12 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc1 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc1 POSTHOOK: Output: database:ice_rest @@ -195,12 +195,12 @@ POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:de POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 4e31d7bef60..82b5e1e1e5a 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -295,6 +295,7 @@ TOK_CREATEMACRO; TOK_DROPMACRO; TOK_TEMPORARY; TOK_CREATEVIEW; +TOK_VIEWMETADATAFORMAT; TOK_DROPVIEW; TOK_ALTERVIEW; TOK_ALTERVIEW_PROPERTIES; @@ -1651,17 +1652,17 @@ createViewStatement : KW_CREATE (orReplace)? KW_VIEW (ifNotExists)? name=tableName (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition? tablePropertiesPrefixed? + (viewMeta=viewMetadataFormat)? KW_AS selectStatementWithCTE - (viewStoredFormat=tableFileFormat)? -> ^(TOK_CREATEVIEW $name orReplace? ifNotExists? columnNameCommentList? tableComment? viewPartition? tablePropertiesPrefixed? + $viewMeta? selectStatementWithCTE - $viewStoredFormat? ) ; @@ -2056,6 +2057,14 @@ tableFileFormat -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; +// Native view metadata: only STORED BY <identifier> (e.g. ICEBERG). No SerDe props or STORED AS tail. +viewMetadataFormat +@init { pushMsg("view metadata format", state); } +@after { popMsg(state); } + : KW_STORED KW_BY handlerId=identifier + -> ^(TOK_VIEWMETADATAFORMAT $handlerId) + ; + tableLocation @init { pushMsg("table location specification", state); } @after { popMsg(state); } diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java index 7a612a0ec41..9e0305bc3e8 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java @@ -93,9 +93,9 @@ public void testParseStructNamedDefault() throws Exception { @Test public void testParseCreateViewStoredByIceberg() throws Exception { ASTNode tree = parseDriver.parse( - "create view v1 as select * from t stored by iceberg", null).getTree(); + "create view v1 stored by iceberg as select * from t", null).getTree(); assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); - assertTrue(tree.dump(), tree.toStringTree().contains("tok_storagehandler")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_viewmetadataformat")); assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 61a19adce57..d750801172a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -47,6 +48,8 @@ */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + private static final String ICEBERG_STORAGE_HANDLER_CLASS = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,16 +78,8 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List<String> partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; - ASTNode storageClause = null; - for (int storageTokenType : new int[] { HiveParser.TOK_STORAGEHANDLER, HiveParser.TOK_TABLEFILEFORMAT, - HiveParser.TOK_FILEFORMAT_GENERIC }) { - ASTNode storageClauseCandidate = children.remove(storageTokenType); - if (storageClauseCandidate != null) { - storageClause = storageClauseCandidate; - break; - } - } - boolean icebergNativeView = validateOptionalViewStorageClause(storageClause); + ASTNode viewMetadataFormat = children.remove(HiveParser.TOK_VIEWMETADATAFORMAT); + boolean icebergNativeView = resolveNativeIcebergView(viewMetadataFormat); assert children.isEmpty(); @@ -203,37 +198,26 @@ private List<FieldSchema> getPartitionColumns(List<String> partitionColumnNames) } /** - * Optional trailing {@code tableFileFormat} on CREATE VIEW: only {@code STORED BY ICEBERG} is allowed - * (no serde properties or {@code STORED AS} tail). + * Native Iceberg view when {@code STORED BY iceberg} is present, or when it is omitted and + * {@code hive.default.storage.handler.class} is the Iceberg storage handler class. */ - private boolean validateOptionalViewStorageClause(ASTNode storageRoot) throws SemanticException { - if (storageRoot == null) { - return false; - } - if (storageRoot.getToken().getType() == HiveParser.TOK_TABLEFILEFORMAT - || storageRoot.getToken().getType() == HiveParser.TOK_FILEFORMAT_GENERIC) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - storageRoot.toStringTree())); - } - if (storageRoot.getToken().getType() != HiveParser.TOK_STORAGEHANDLER) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - storageRoot.toStringTree())); - } - if (storageRoot.getChildCount() != 1) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY ICEBERG views do not support WITH SERDEPROPERTIES or STORED AS")); - } - ASTNode handlerNode = (ASTNode) storageRoot.getChild(0); - if (handlerNode.getToken().getType() == HiveParser.StringLiteral) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY with string literal handler")); + private boolean resolveNativeIcebergView(ASTNode viewMetadataFormat) throws SemanticException { + if (viewMetadataFormat != null) { + if (viewMetadataFormat.getChildCount() != 1) { + throw new SemanticException("Internal error: expected single handler identifier in view metadata"); + } + String handler = ((ASTNode) viewMetadataFormat.getChild(0)).getText(); + if (handler == null || !handler.equalsIgnoreCase("iceberg")) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( + "Only STORED BY ICEBERG is supported for native views, got: " + handler)); + } + return true; } - String handler = handlerNode.getText(); - if (handler == null || !handler.equalsIgnoreCase("iceberg")) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY " + handler)); + String defaultHandler = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); + if (defaultHandler == null || defaultHandler.isEmpty()) { + return false; } - return true; + return ICEBERG_STORAGE_HANDLER_CLASS.equals(defaultHandler.trim()); } private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException {
