This is an automated email from the ASF dual-hosted git repository. difin pushed a commit to branch iceberg_native_views in repository https://gitbox.apache.org/repos/asf/hive.git
commit 4671f16f1a9076adf603a1d411b667d6f3f6d08d Author: Dmitriy Fingerman <[email protected]> AuthorDate: Mon Apr 27 14:20:47 2026 -0400 fixes --- .../test/queries/positive/iceberg_native_view.q | 10 +++- .../positive/iceberg_rest_catalog_gravitino.q | 6 +-- .../queries/positive/iceberg_rest_catalog_hms.q | 6 +-- .../positive/llap/iceberg_native_view.q.out | 39 +++++++++++++- .../llap/iceberg_rest_catalog_gravitino.q.out | 8 +-- .../positive/llap/iceberg_rest_catalog_hms.q.out | 8 +-- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 13 ++++- .../hadoop/hive/ql/parse/TestParseDefault.java | 4 +- .../ql/ddl/view/create/CreateViewAnalyzer.java | 60 ++++++++-------------- 9 files changed, 95 insertions(+), 59 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q index 4c954f786f0..fb05224dced 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q @@ -28,11 +28,19 @@ update src_ice set last_name = 'ln7a' where first_name='fn7'; delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a'); -create view v_ice as select * from src_ice stored by iceberg; +-- Explicit native view metadata (STORED BY before AS) +create view v_ice stored by iceberg as select * from src_ice; + +-- Native view from default storage handler when STORED BY is omitted +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; +create view v_def as select * from src_ice; +set hive.default.storage.handler.class=; select * from v_ice; +select * from v_def; drop view v_ice; +drop view v_def; drop table src_ice; use default; drop database ice_native_view_db cascade; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 6b73ff2aa13..644097320dd 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -73,14 +73,14 @@ describe formatted ice_orc2; select * from ice_orc2; ----------------------------------------------------------------------------- ---! Iceberg native views (STORED BY ICEBERG) on REST catalog +--! Iceberg native views (STORED BY ICEBERG before AS) on REST catalog ----------------------------------------------------------------------------- -- Enable once CBO supports Iceberg native views on partitioned tables set hive.cbo.enable=false; -create view ice_v1 as select * from ice_orc1 stored by iceberg; -create view ice_v2 as select * from ice_orc2 stored by iceberg; +create view ice_v1 stored by iceberg as select * from ice_orc1; +create view ice_v2 stored by iceberg as select * from ice_orc2; select * from ice_v1; select * from ice_v2; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index eefd5bd5ce9..c282489a6f3 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -68,14 +68,14 @@ describe formatted ice_orc2; select * from ice_orc2; ----------------------------------------------------------------------------- ---! Iceberg native views (STORED BY ICEBERG) on REST catalog +--! Iceberg native views (STORED BY ICEBERG before AS) on REST catalog ----------------------------------------------------------------------------- -- Enable once CBO supports Iceberg native views on partitioned tables set hive.cbo.enable=false; -create view ice_v1 as select * from ice_orc1 stored by iceberg; -create view ice_v2 as select * from ice_orc2 stored by iceberg; +create view ice_v1 stored by iceberg as select * from ice_orc1; +create view ice_v2 stored by iceberg as select * from ice_orc2; select * from ice_v1; select * from ice_v2; diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out index ae1a55ef129..5fb5561062c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out @@ -162,12 +162,12 @@ POSTHOOK: query: delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a') POSTHOOK: type: QUERY POSTHOOK: Input: ice_native_view_db@src_ice #### A masked pattern was here #### -PREHOOK: query: create view v_ice as select * from src_ice stored by iceberg +PREHOOK: query: create view v_ice stored by iceberg as select * from src_ice PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_native_view_db@src_ice PREHOOK: Output: database:ice_native_view_db PREHOOK: Output: ice_native_view_db@v_ice -POSTHOOK: query: create view v_ice as select * from src_ice stored by iceberg +POSTHOOK: query: create view v_ice stored by iceberg as select * from src_ice POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_native_view_db@src_ice POSTHOOK: Output: database:ice_native_view_db @@ -175,6 +175,19 @@ POSTHOOK: Output: ice_native_view_db@v_ice POSTHOOK: Lineage: v_ice.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] POSTHOOK: Lineage: v_ice.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] POSTHOOK: Lineage: v_ice.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: create view v_def as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view v_def as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] PREHOOK: query: select * from v_ice PREHOOK: type: QUERY PREHOOK: Input: ice_native_view_db@src_ice @@ -189,6 +202,20 @@ fn3 ln3a 1 fn4 ln4a 1 fn5 ln5a 2 fn6 ln6a 2 +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 PREHOOK: query: drop view v_ice PREHOOK: type: DROPVIEW PREHOOK: Input: ice_native_view_db@v_ice @@ -197,6 +224,14 @@ POSTHOOK: query: drop view v_ice POSTHOOK: type: DROPVIEW POSTHOOK: Input: ice_native_view_db@v_ice POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def PREHOOK: query: drop table src_ice PREHOOK: type: DROPTABLE PREHOOK: Input: ice_native_view_db@src_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 0c27b278031..e76bf23e009 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -180,12 +180,12 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc1 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc1 POSTHOOK: Output: database:ice_rest @@ -195,12 +195,12 @@ POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:de POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 3a046d00a7f..b42301b58dd 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -180,12 +180,12 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc1 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 as select * from ice_orc1 stored by iceberg +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc1 POSTHOOK: Output: database:ice_rest @@ -195,12 +195,12 @@ POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:de POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 as select * from ice_orc2 stored by iceberg +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 4e31d7bef60..82b5e1e1e5a 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -295,6 +295,7 @@ TOK_CREATEMACRO; TOK_DROPMACRO; TOK_TEMPORARY; TOK_CREATEVIEW; +TOK_VIEWMETADATAFORMAT; TOK_DROPVIEW; TOK_ALTERVIEW; TOK_ALTERVIEW_PROPERTIES; @@ -1651,17 +1652,17 @@ createViewStatement : KW_CREATE (orReplace)? KW_VIEW (ifNotExists)? name=tableName (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition? tablePropertiesPrefixed? + (viewMeta=viewMetadataFormat)? KW_AS selectStatementWithCTE - (viewStoredFormat=tableFileFormat)? -> ^(TOK_CREATEVIEW $name orReplace? ifNotExists? columnNameCommentList? tableComment? viewPartition? tablePropertiesPrefixed? + $viewMeta? selectStatementWithCTE - $viewStoredFormat? ) ; @@ -2056,6 +2057,14 @@ tableFileFormat -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; +// Native view metadata: only STORED BY <identifier> (e.g. ICEBERG). No SerDe props or STORED AS tail. +viewMetadataFormat +@init { pushMsg("view metadata format", state); } +@after { popMsg(state); } + : KW_STORED KW_BY handlerId=identifier + -> ^(TOK_VIEWMETADATAFORMAT $handlerId) + ; + tableLocation @init { pushMsg("table location specification", state); } @after { popMsg(state); } diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java index 7a612a0ec41..9e0305bc3e8 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java @@ -93,9 +93,9 @@ public void testParseStructNamedDefault() throws Exception { @Test public void testParseCreateViewStoredByIceberg() throws Exception { ASTNode tree = parseDriver.parse( - "create view v1 as select * from t stored by iceberg", null).getTree(); + "create view v1 stored by iceberg as select * from t", null).getTree(); assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); - assertTrue(tree.dump(), tree.toStringTree().contains("tok_storagehandler")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_viewmetadataformat")); assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 61a19adce57..d750801172a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -47,6 +48,8 @@ */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + private static final String ICEBERG_STORAGE_HANDLER_CLASS = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,16 +78,8 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List<String> partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; - ASTNode storageClause = null; - for (int storageTokenType : new int[] { HiveParser.TOK_STORAGEHANDLER, HiveParser.TOK_TABLEFILEFORMAT, - HiveParser.TOK_FILEFORMAT_GENERIC }) { - ASTNode storageClauseCandidate = children.remove(storageTokenType); - if (storageClauseCandidate != null) { - storageClause = storageClauseCandidate; - break; - } - } - boolean icebergNativeView = validateOptionalViewStorageClause(storageClause); + ASTNode viewMetadataFormat = children.remove(HiveParser.TOK_VIEWMETADATAFORMAT); + boolean icebergNativeView = resolveNativeIcebergView(viewMetadataFormat); assert children.isEmpty(); @@ -203,37 +198,26 @@ private List<FieldSchema> getPartitionColumns(List<String> partitionColumnNames) } /** - * Optional trailing {@code tableFileFormat} on CREATE VIEW: only {@code STORED BY ICEBERG} is allowed - * (no serde properties or {@code STORED AS} tail). + * Native Iceberg view when {@code STORED BY iceberg} is present, or when it is omitted and + * {@code hive.default.storage.handler.class} is the Iceberg storage handler class. */ - private boolean validateOptionalViewStorageClause(ASTNode storageRoot) throws SemanticException { - if (storageRoot == null) { - return false; - } - if (storageRoot.getToken().getType() == HiveParser.TOK_TABLEFILEFORMAT - || storageRoot.getToken().getType() == HiveParser.TOK_FILEFORMAT_GENERIC) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - storageRoot.toStringTree())); - } - if (storageRoot.getToken().getType() != HiveParser.TOK_STORAGEHANDLER) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - storageRoot.toStringTree())); - } - if (storageRoot.getChildCount() != 1) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY ICEBERG views do not support WITH SERDEPROPERTIES or STORED AS")); - } - ASTNode handlerNode = (ASTNode) storageRoot.getChild(0); - if (handlerNode.getToken().getType() == HiveParser.StringLiteral) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY with string literal handler")); + private boolean resolveNativeIcebergView(ASTNode viewMetadataFormat) throws SemanticException { + if (viewMetadataFormat != null) { + if (viewMetadataFormat.getChildCount() != 1) { + throw new SemanticException("Internal error: expected single handler identifier in view metadata"); + } + String handler = ((ASTNode) viewMetadataFormat.getChild(0)).getText(); + if (handler == null || !handler.equalsIgnoreCase("iceberg")) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( + "Only STORED BY ICEBERG is supported for native views, got: " + handler)); + } + return true; } - String handler = handlerNode.getText(); - if (handler == null || !handler.equalsIgnoreCase("iceberg")) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "STORED BY " + handler)); + String defaultHandler = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); + if (defaultHandler == null || defaultHandler.isEmpty()) { + return false; } - return true; + return ICEBERG_STORAGE_HANDLER_CLASS.equals(defaultHandler.trim()); } private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException {
