This is an automated email from the ASF dual-hosted git repository. difin pushed a commit to branch iceberg_native_views in repository https://gitbox.apache.org/repos/asf/hive.git
commit 14521074c55f971dc3db17eab748f72d56d48e1d Author: Dmitriy Fingerman <[email protected]> AuthorDate: Thu Apr 23 15:49:20 2026 -0400 HIVE-29578: Iceberg: support for Iceberg native views --- .../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 2 + .../apache/iceberg/hive/HiveViewOperations.java | 2 +- .../org/apache/iceberg/hive/MetastoreUtil.java | 73 ++++ .../iceberg/hive/NativeIcebergViewSupport.java | 124 +++++++ .../iceberg/hive/client/HiveRESTCatalogClient.java | 49 ++- .../iceberg/hive/TestNativeIcebergViewSupport.java | 136 ++++++++ .../test/queries/positive/iceberg_native_view.q | 62 ++++ .../positive/iceberg_rest_catalog_gravitino.q | 53 ++- .../queries/positive/iceberg_rest_catalog_hms.q | 53 ++- .../positive/llap/iceberg_native_view.q.out | 373 +++++++++++++++++++++ .../llap/iceberg_rest_catalog_gravitino.q.out | 292 ++++++++++++++++ .../positive/llap/iceberg_rest_catalog_hms.q.out | 290 ++++++++++++++++ .../hive/TestHiveRESTCatalogClientITBase.java | 52 ++- .../test/resources/testconfiguration.properties | 2 + .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 11 + .../hadoop/hive/ql/parse/TestParseDefault.java | 9 + .../ql/ddl/view/create/CreateViewAnalyzer.java | 31 +- .../hive/ql/ddl/view/create/CreateViewDesc.java | 17 + .../ql/ddl/view/create/CreateViewOperation.java | 126 +++++++ 19 files changed, 1739 insertions(+), 18 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 57f7cadac18..de96936df4c 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -445,6 +445,8 @@ public enum ErrorMsg { @Deprecated // kept for backwards reference REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true), REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true), + VIEW_STORAGE_HANDLER_UNSUPPORTED(10448, "CREATE VIEW only supports STORED BY ICEBERG for native " + + "Iceberg views; unsupported storage clause: {0}", true), UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"), MATERIALIZED_VIEW_DEF_EMPTY(10403, "Query for the materialized view rebuild could not be retrieved"), MERGE_PREDIACTE_REQUIRED(10404, "MERGE statement with both UPDATE and DELETE clauses " + diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java index c0c959974a6..10f6736c353 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java @@ -302,7 +302,7 @@ private Table newHMSView(ViewMetadata metadata) { tableType().name()); } - private String sqlFor(ViewMetadata metadata) { + public static String sqlFor(ViewMetadata metadata) { SQLViewRepresentation closest = null; for (ViewRepresentation representation : metadata.currentVersion().representations()) { if (representation instanceof SQLViewRepresentation) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 95e1e5b3662..7330d5ba863 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -46,6 +46,11 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.SQLViewRepresentation; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; import org.apache.thrift.TException; public class MetastoreUtil { @@ -148,6 +153,74 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co return result; } + /** + * Builds a Hive metastore {@link Table} representation for an Iceberg {@link View}, for clients + * (e.g. {@code HiveRESTCatalogClient}) that bridge Iceberg catalog metadata into the HMS API. + */ + public static Table toHiveView(View view, Configuration conf) { + Table result = new Table(); + TableName tableName = + TableName.fromString( + view.name(), MetaStoreUtils.getDefaultCatalog(conf), Warehouse.DEFAULT_DATABASE_NAME); + result.setCatName(tableName.getCat()); + result.setDbName(tableName.getDb()); + result.setTableName(tableName.getTable()); + result.setTableType(TableType.VIRTUAL_VIEW.toString()); + + ViewMetadata metadata = ((BaseView) view).operations().current(); + String sqlText = viewSqlText(view, metadata); + result.setViewOriginalText(sqlText); + result.setViewExpandedText(sqlText); + + long nowMillis = System.currentTimeMillis(); + int nowSec = (int) (nowMillis / 1000); + String owner = + PropertyUtil.propertyAsString( + metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, System.getProperty("user.name")); + result.setOwner(owner); + result.setCreateTime(nowSec); + result.setLastAccessTime(nowSec); + result.setRetention(Integer.MAX_VALUE); + + boolean hiveEngineEnabled = false; + result.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + StorageDescriptor sd = result.getSd(); + + if (sd.getBucketCols() == null) { + sd.setBucketCols(Lists.newArrayList()); + } + + if (sd.getSortCols() == null) { + sd.setSortCols(Lists.newArrayList()); + } + + long maxHiveTablePropertySize = + conf.getLong( + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); + HMSTablePropertyHelper.updateHmsTableForIcebergView( + metadata.metadataFileLocation(), + result, + metadata, + Collections.emptySet(), + maxHiveTablePropertySize, + null); + + String catalogType = IcebergCatalogProperties.getCatalogType(conf); + if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { + result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + } + return result; + } + + private static String viewSqlText(View view, ViewMetadata metadata) { + SQLViewRepresentation hiveRepr = view.sqlFor("hive"); + if (hiveRepr != null) { + return hiveRepr.sql(); + } + return HiveViewOperations.sqlFor(metadata); + } + private static StorageDescriptor getHiveStorageDescriptor(org.apache.iceberg.Table table) { var result = new StorageDescriptor(); result.setCols(HiveSchemaUtil.convert(table.schema())); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java new file mode 100644 index 00000000000..ace4ea703b8 --- /dev/null +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.io.Closeable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.view.ViewBuilder; + +/** + * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST + * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. + */ +public final class NativeIcebergViewSupport { + + /** HMS parameter aligned with Hive's {@code CreateViewDesc#ICEBERG_NATIVE_VIEW_PROPERTY}. */ + public static final String ICEBERG_NATIVE_VIEW_PROPERTY = "hive.iceberg.native.view"; + + private NativeIcebergViewSupport() { + } + + /** + * Creates or replaces a view in the Iceberg catalog. + * + * @return {@code false} if skipped because {@code ifNotExists} is true and the view already exists + */ + public static boolean createOrReplaceNativeView(Configuration conf, String databaseName, String viewName, + List<FieldSchema> fieldSchemas, String viewSql, Map<String, String> tblProperties, String comment, + boolean replace, boolean ifNotExists) throws Exception { + + TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map<String, String> catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + try { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + if (!replace && ifNotExists && viewCatalog.viewExists(identifier)) { + return false; + } + + ViewBuilder builder = startViewBuilder(viewCatalog, identifier, fieldSchemas, viewSql); + builder = applyCommentAndTblProps(builder, tblProperties, comment); + commitView(builder, replace); + return true; + } finally { + if (catalog instanceof Closeable) { + ((Closeable) catalog).close(); + } + } + } + + private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { + if (!(catalog instanceof ViewCatalog)) { + throw new UnsupportedOperationException( + String.format( + "Iceberg catalog '%s' does not implement ViewCatalog.", + catalogName) + + " Native views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); + } + return (ViewCatalog) catalog; + } + + private static ViewBuilder startViewBuilder( + ViewCatalog viewCatalog, + TableIdentifier identifier, + List<FieldSchema> fieldSchemas, + String viewSql) { + return viewCatalog + .buildView(identifier) + .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) + .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) + .withQuery("hive", viewSql) + .withProperty(ICEBERG_NATIVE_VIEW_PROPERTY, "true"); + } + + private static ViewBuilder applyCommentAndTblProps( + ViewBuilder builder, Map<String, String> tblProperties, String comment) { + ViewBuilder viewBuilder = builder; + if (comment != null && !comment.isEmpty()) { + viewBuilder = viewBuilder.withProperty("comment", comment); + } + if (tblProperties != null) { + for (Map.Entry<String, String> e : tblProperties.entrySet()) { + if (e.getKey() != null && e.getValue() != null) { + viewBuilder = viewBuilder.withProperty(e.getKey(), e.getValue()); + } + } + } + return viewBuilder; + } + + private static void commitView(ViewBuilder builder, boolean replace) { + if (replace) { + builder.createOrReplace(); + } else { + builder.create(); + } + } +} diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 4390d5a0bca..2194b9950e3 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -21,10 +21,12 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; @@ -43,7 +45,9 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.hive.HMSTablePropertyHelper; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; @@ -54,6 +58,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.View; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,10 +128,19 @@ public List<String> getTables(String catName, String dbName, String tablePattern Pattern pattern = Pattern.compile(regex); // List tables from the specific database (namespace) and filter them. - return restCatalog.listTables(Namespace.of(dbName)).stream() + Set<String> names = new LinkedHashSet<>(); + restCatalog.listTables(Namespace.of(dbName)).stream() .map(TableIdentifier::name) .filter(pattern.asPredicate()) - .toList(); + .forEach(names::add); + if (restCatalog instanceof ViewCatalog) { + ((ViewCatalog) restCatalog) + .listViews(Namespace.of(dbName)).stream() + .map(TableIdentifier::name) + .filter(pattern.asPredicate()) + .forEach(names::add); + } + return Lists.newArrayList(names); } @Override @@ -136,7 +150,12 @@ public List<String> getAllTables(String catName, String dbName) { @Override public void dropTable(Table table, boolean deleteData, boolean ignoreUnknownTab, boolean ifPurge) throws TException { - restCatalog.dropTable(TableIdentifier.of(table.getDbName(), table.getTableName())); + TableIdentifier id = TableIdentifier.of(table.getDbName(), table.getTableName()); + if (restCatalog instanceof ViewCatalog && ((ViewCatalog) restCatalog).viewExists(id)) { + ((ViewCatalog) restCatalog).dropView(id); + } else { + restCatalog.dropTable(id); + } } private void validateCurrentCatalog(String catName) { @@ -149,7 +168,11 @@ private void validateCurrentCatalog(String catName) { @Override public boolean tableExists(String catName, String dbName, String tableName) { validateCurrentCatalog(catName); - return restCatalog.tableExists(TableIdentifier.of(dbName, tableName)); + TableIdentifier id = TableIdentifier.of(dbName, tableName); + if (restCatalog.tableExists(id)) { + return true; + } + return restCatalog instanceof ViewCatalog && ((ViewCatalog) restCatalog).viewExists(id); } @Override @@ -178,14 +201,22 @@ public Database getDatabase(String catName, String dbName) throws NoSuchObjectEx @Override public Table getTable(GetTableRequest tableRequest) throws TException { validateCurrentCatalog(tableRequest.getCatName()); - org.apache.iceberg.Table icebergTable; + TableIdentifier id = + TableIdentifier.of(tableRequest.getDbName(), tableRequest.getTblName()); try { - icebergTable = restCatalog.loadTable(TableIdentifier.of(tableRequest.getDbName(), - tableRequest.getTblName())); - } catch (NoSuchTableException exception) { + org.apache.iceberg.Table icebergTable = restCatalog.loadTable(id); + return MetastoreUtil.toHiveTable(icebergTable, conf); + } catch (NoSuchTableException tableMissing) { + if (restCatalog instanceof ViewCatalog) { + try { + View icebergView = ((ViewCatalog) restCatalog).loadView(id); + return MetastoreUtil.toHiveView(icebergView, conf); + } catch (NoSuchViewException viewMissing) { + throw new NoSuchObjectException(); + } + } throw new NoSuchObjectException(); } - return MetastoreUtil.toHiveTable(icebergTable, conf); } @Override diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java new file mode 100644 index 00000000000..daf187b5c09 --- /dev/null +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.View; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE; +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestNativeIcebergViewSupport { + + private static final String DB = "native_vw_db"; + private static final String VIEW = "native_vw"; + + @RegisterExtension + private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION = + HiveMetastoreExtension.builder().withDatabase(DB).build(); + + @AfterEach + public void dropView() { + HiveCatalog cat = verifyCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + if (cat.viewExists(id)) { + cat.dropView(id); + } + } + + private HiveConf nativeViewConf() { + HiveConf conf = new HiveConf(HIVE_METASTORE_EXTENSION.hiveConf()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT, "hive"); + conf.set( + IcebergCatalogProperties.catalogPropertyConfigKey("hive", ICEBERG_CATALOG_TYPE), + ICEBERG_CATALOG_TYPE_HIVE); + return conf; + } + + private HiveCatalog verifyCatalog() { + return (HiveCatalog) + CatalogUtil.loadCatalog( + HiveCatalog.class.getName(), + ICEBERG_CATALOG_TYPE_HIVE, + ImmutableMap.of( + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, + String.valueOf(TimeUnit.SECONDS.toMillis(10))), + HIVE_METASTORE_EXTENSION.hiveConf()); + } + + @Test + public void testCreateCommitsNativeViewWithMarkerProperty() throws Exception { + HiveConf conf = nativeViewConf(); + List<FieldSchema> cols = + Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)); + String sql = String.format("select id, name from %s.src_tbl", DB); + Map<String, String> props = Collections.singletonMap("k1", "v1"); + + boolean created = + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, sql, props, "hello-view", false, false); + assertThat(created).isTrue(); + + HiveCatalog cat = verifyCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + assertThat(cat.viewExists(id)).isTrue(); + View view = cat.loadView(id); + assertThat(view.properties().get(NativeIcebergViewSupport.ICEBERG_NATIVE_VIEW_PROPERTY)) + .isEqualTo("true"); + assertThat(view.properties().get("comment")).isEqualTo("hello-view"); + assertThat(view.properties().get("k1")).isEqualTo("v1"); + HiveViewOperations ops = (HiveViewOperations) ((BaseView) view).operations(); + assertThat(ops.current().currentVersion().representations()).isNotEmpty(); + } + + @Test + public void testIfNotExistsReturnsFalseWhenViewExists() throws Exception { + HiveConf conf = nativeViewConf(); + List<FieldSchema> cols = Collections.singletonList(new FieldSchema("id", "int", null)); + + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false)) + .isTrue(); + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, false, true)) + .isFalse(); + } + + @Test + public void testReplaceUpdatesView() throws Exception { + HiveConf conf = nativeViewConf(); + List<FieldSchema> cols = Collections.singletonList(new FieldSchema("id", "int", null)); + + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, true, false)) + .isTrue(); + + assertThat(verifyCatalog().viewExists(TableIdentifier.of(DB, VIEW))).isTrue(); + } +} diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q new file mode 100644 index 00000000000..eea5657baae --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q @@ -0,0 +1,62 @@ +-- SORT_QUERY_RESULTS +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ + +create database ice_native_view_db; +use ice_native_view_db; + +create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc; + +insert into src_ice values ('fn1','ln1', 1); +insert into src_ice values ('fn2','ln2', 1); +insert into src_ice values ('fn3','ln3', 1); +insert into src_ice values ('fn4','ln4', 1); +insert into src_ice values ('fn5','ln5', 2); +insert into src_ice values ('fn6','ln6', 2); +insert into src_ice values ('fn7','ln7', 2); + +update src_ice set last_name = 'ln1a' where first_name='fn1'; +update src_ice set last_name = 'ln2a' where first_name='fn2'; +update src_ice set last_name = 'ln3a' where first_name='fn3'; +update src_ice set last_name = 'ln4a' where first_name='fn4'; +update src_ice set last_name = 'ln5a' where first_name='fn5'; +update src_ice set last_name = 'ln6a' where first_name='fn6'; +update src_ice set last_name = 'ln7a' where first_name='fn7'; + +delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a'); + +---------------------------------------------------------------- +-- Explicit native view metadata (STORED BY before AS) +---------------------------------------------------------------- + +create view v_ice stored by iceberg as select * from src_ice; +select * from v_ice; +desc formatted v_ice; +drop view v_ice; + +---------------------------------------------------------------- +-- Regular Hive view from when STORED BY is HiveIcebergStorageHandler +---------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view v_def as select * from src_ice; +select * from v_def; +desc formatted v_def; +drop view v_def; + +---------------------------------------------------------------- +-- Native view from default storage handler when STORED BY is omitted +---------------------------------------------------------------- + +set hive.default.storage.handler.class=; + +create view v_hive as select * from src_ice; +select * from v_hive; +desc formatted v_hive; +drop view v_hive; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 81982ca44d9..6974ea9ad84 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -74,6 +72,55 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +-- Enable once CBO supports Iceberg native views on partitioned tables +set hive.cbo.enable=false; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 stored by iceberg as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 stored by iceberg as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 stored by iceberg as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 stored by iceberg as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index 27f23122240..52fd5fcbea6 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -69,6 +67,55 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +-- Enable once CBO supports Iceberg native views on partitioned tables +set hive.cbo.enable=false; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 stored by iceberg as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 stored by iceberg as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 stored by iceberg as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 stored by iceberg as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out new file mode 100644 index 00000000000..449f9f39f66 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out @@ -0,0 +1,373 @@ +PREHOOK: query: create database ice_native_view_db +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:ice_native_view_db +POSTHOOK: query: create database ice_native_view_db +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:ice_native_view_db +PREHOOK: query: use ice_native_view_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:ice_native_view_db +POSTHOOK: query: use ice_native_view_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:ice_native_view_db +PREHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn1','ln1', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn1','ln1', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn2','ln2', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn2','ln2', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn3','ln3', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn3','ln3', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn4','ln4', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn4','ln4', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn5','ln5', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn5','ln5', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn6','ln6', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn6','ln6', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: insert into src_ice values ('fn7','ln7', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: insert into src_ice values ('fn7','ln7', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln1a' where first_name='fn1' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln1a' where first_name='fn1' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln2a' where first_name='fn2' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln2a' where first_name='fn2' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln3a' where first_name='fn3' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln3a' where first_name='fn3' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln4a' where first_name='fn4' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln4a' where first_name='fn4' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln5a' where first_name='fn5' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln5a' where first_name='fn5' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln6a' where first_name='fn6' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln6a' where first_name='fn6' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: update src_ice set last_name = 'ln7a' where first_name='fn7' +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: update src_ice set last_name = 'ln7a' where first_name='fn7' +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a') +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +#### A masked pattern was here #### +POSTHOOK: query: delete from src_ice where last_name in ('ln1a', 'ln2a', 'ln7a') +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +#### A masked pattern was here #### +PREHOOK: query: create view v_ice stored by iceberg as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create view v_ice stored by iceberg as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: Lineage: v_ice.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_ice.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_ice.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +#### A masked pattern was here #### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_ice +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: drop view v_ice +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: create view v_def as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view v_def as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 +PREHOOK: query: desc formatted v_def +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_def +POSTHOOK: query: desc formatted v_def +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_def +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: create view v_hive as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: create view v_hive as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: Lineage: v_hive.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_hive.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_hive.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_hive +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +POSTHOOK: query: select * from v_hive +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +fn3 ln3a 1 +fn4 ln4a 1 +fn5 ln5a 2 +fn6 ln6a 2 +PREHOOK: query: desc formatted v_hive +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: query: desc formatted v_hive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_hive +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select * from src_ice +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_hive +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: drop view v_hive +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: ice_native_view_db@v_hive diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 8bba659e8fd..3e50a68ef90 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -180,6 +180,296 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest @@ -218,6 +508,7 @@ PREHOOK: query: show databases PREHOOK: type: SHOWDATABASES POSTHOOK: query: show databases POSTHOOK: type: SHOWDATABASES +default ice_rest PREHOOK: query: drop database ice_rest PREHOOK: type: DROPDATABASE @@ -231,3 +522,4 @@ PREHOOK: query: show databases PREHOOK: type: SHOWDATABASES POSTHOOK: query: show databases POSTHOOK: type: SHOWDATABASES +default diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 409eb484480..3e50a68ef90 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -180,6 +180,296 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.iceberg.native.view true +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index fd30223c993..6ba5ef472e2 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -38,13 +39,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.NativeIcebergViewSupport; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -67,11 +70,14 @@ public abstract class TestHiveRESTCatalogClientITBase { static final String DB_NAME = "ice_db"; + static final String VIEW_DB_NAME = "ice_db_view"; + static final String NATIVE_VIEW_NAME = "native_rest_v"; static final String TABLE_NAME = "ice_tbl"; static final String CATALOG_NAME = "ice01"; static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; static final String REST_CATALOG_PREFIX = String.format("%s%s.", IcebergCatalogProperties.CATALOG_CONFIG_PREFIX, CATALOG_NAME); + static final String ICEBERG_VIEW_TYPE_VALUE = "iceberg-view"; HiveConf hiveConf; Configuration conf; @@ -195,6 +201,50 @@ public void testIceberg() throws Exception { Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME)); } + @Test + public void testNativeIcebergView() throws Exception { + Database db = new Database(); + db.setCatalogName(CATALOG_NAME); + db.setName(VIEW_DB_NAME); + db.setOwnerType(PrincipalType.USER); + db.setOwnerName(System.getProperty("user.name")); + String warehouseDir = MetastoreConf.get(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); + db.setLocationUri(warehouseDir + "/" + VIEW_DB_NAME + ".db"); + hive.createDatabase(db, true); + + List<FieldSchema> cols = Collections.singletonList(new FieldSchema("x", "int", "")); + NativeIcebergViewSupport.createOrReplaceNativeView( + hiveConf, + VIEW_DB_NAME, + NATIVE_VIEW_NAME, + cols, + "select 1 as x", + null, + "rest-native-view", + false, + false); + + GetTableRequest getTableRequest = new GetTableRequest(); + getTableRequest.setCatName(CATALOG_NAME); + getTableRequest.setDbName(VIEW_DB_NAME); + getTableRequest.setTblName(NATIVE_VIEW_NAME); + Table view = msClient.getTable(getTableRequest); + Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); + String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); + Assertions.assertNotNull(tableTypeProp); + Assertions.assertEquals(ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); + + Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + List<String> names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); + Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); + + msClient.dropTable(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME); + Assertions.assertFalse(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + msClient.dropDatabase(VIEW_DB_NAME); + } + private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, Map<String, String> tableParameters) throws Exception { db.dropTable(catName, dbName, tableName); diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 64a761acfdd..fd38e9c1f2a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -433,6 +433,7 @@ iceberg.llap.query.files=\ iceberg_create_locally_zordered_table.q,\ iceberg_merge_delete_files.q,\ iceberg_merge_files.q,\ + iceberg_native_view.q,\ llap_iceberg_read_orc.q,\ llap_iceberg_read_parquet.q,\ puffin_col_stats_with_time_travel.q,\ @@ -484,6 +485,7 @@ iceberg.llap.only.query.files=\ iceberg_create_locally_zordered_table.q,\ iceberg_merge_delete_files.q,\ iceberg_merge_files.q,\ + iceberg_native_view.q,\ llap_iceberg_read_orc.q,\ llap_iceberg_read_parquet.q,\ puffin_col_stats_with_time_travel.q diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 0033aa3251c..82b5e1e1e5a 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -295,6 +295,7 @@ TOK_CREATEMACRO; TOK_DROPMACRO; TOK_TEMPORARY; TOK_CREATEVIEW; +TOK_VIEWMETADATAFORMAT; TOK_DROPVIEW; TOK_ALTERVIEW; TOK_ALTERVIEW_PROPERTIES; @@ -1651,6 +1652,7 @@ createViewStatement : KW_CREATE (orReplace)? KW_VIEW (ifNotExists)? name=tableName (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition? tablePropertiesPrefixed? + (viewMeta=viewMetadataFormat)? KW_AS selectStatementWithCTE -> ^(TOK_CREATEVIEW $name orReplace? @@ -1659,6 +1661,7 @@ createViewStatement tableComment? viewPartition? tablePropertiesPrefixed? + $viewMeta? selectStatementWithCTE ) ; @@ -2054,6 +2057,14 @@ tableFileFormat -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; +// Native view metadata: only STORED BY <identifier> (e.g. ICEBERG). No SerDe props or STORED AS tail. +viewMetadataFormat +@init { pushMsg("view metadata format", state); } +@after { popMsg(state); } + : KW_STORED KW_BY handlerId=identifier + -> ^(TOK_VIEWMETADATAFORMAT $handlerId) + ; + tableLocation @init { pushMsg("table location specification", state); } @after { popMsg(state); } diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java index 45a3a9e7418..9e0305bc3e8 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java @@ -90,6 +90,15 @@ public void testParseStructNamedDefault() throws Exception { assertFalse(tree.dump(), tree.toStringTree().contains("tok_default_value")); } + @Test + public void testParseCreateViewStoredByIceberg() throws Exception { + ASTNode tree = parseDriver.parse( + "create view v1 stored by iceberg as select * from t", null).getTree(); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_viewmetadataformat")); + assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); + } + @Test public void testParseStructFieldNamedDefault() throws Exception { ASTNode tree = parseDriver.parse( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 60b123dbb7b..d750801172a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -47,6 +48,8 @@ */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + private static final String ICEBERG_STORAGE_HANDLER_CLASS = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,6 +78,9 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List<String> partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; + ASTNode viewMetadataFormat = children.remove(HiveParser.TOK_VIEWMETADATAFORMAT); + boolean icebergNativeView = resolveNativeIcebergView(viewMetadataFormat); + assert children.isEmpty(); if (ifNotExists && orReplace) { @@ -94,7 +100,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List<FieldSchema> partitionColumns = getPartitionColumns(partitionColumnNames); setColumnAccessInfo(analyzer.getColumnAccessInfo()); CreateViewDesc desc = new CreateViewDesc(fqViewName, schema, comment, properties, partitionColumnNames, - ifNotExists, orReplace, originalText, expandedText, partitionColumns); + ifNotExists, orReplace, originalText, expandedText, partitionColumns, icebergNativeView); validateCreateView(desc, analyzer); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); @@ -191,6 +197,29 @@ private List<FieldSchema> getPartitionColumns(List<String> partitionColumnNames) return partitionColumnsCopy; } + /** + * Native Iceberg view when {@code STORED BY iceberg} is present, or when it is omitted and + * {@code hive.default.storage.handler.class} is the Iceberg storage handler class. + */ + private boolean resolveNativeIcebergView(ASTNode viewMetadataFormat) throws SemanticException { + if (viewMetadataFormat != null) { + if (viewMetadataFormat.getChildCount() != 1) { + throw new SemanticException("Internal error: expected single handler identifier in view metadata"); + } + String handler = ((ASTNode) viewMetadataFormat.getChild(0)).getText(); + if (handler == null || !handler.equalsIgnoreCase("iceberg")) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( + "Only STORED BY ICEBERG is supported for native views, got: " + handler)); + } + return true; + } + String defaultHandler = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); + if (defaultHandler == null || defaultHandler.isEmpty()) { + return false; + } + return ICEBERG_STORAGE_HANDLER_CLASS.equals(defaultHandler.trim()); + } + private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { try { validateTablesUsed(analyzer); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index e71cbce773f..bbdf0df0da2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -34,12 +34,16 @@ public class CreateViewDesc extends AbstractCreateViewDesc { private static final long serialVersionUID = 1L; + /** HMS table property set when the view is declared with {@code STORED BY ICEBERG} (native Iceberg view). */ + public static final String ICEBERG_NATIVE_VIEW_PROPERTY = "hive.iceberg.native.view"; + private final String comment; private final Map<String, String> properties; private final List<String> partitionColumnNames; private final boolean ifNotExists; private final boolean replace; private final List<FieldSchema> partitionColumns; + private final boolean icebergNativeView; private ReplicationSpec replicationSpec = null; private String ownerName = null; @@ -47,6 +51,13 @@ public class CreateViewDesc extends AbstractCreateViewDesc { public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, Map<String, String> properties, List<String> partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, String expandedText, List<FieldSchema> partitionColumns) { + this(viewName, schema, comment, properties, partitionColumnNames, ifNotExists, replace, originalText, + expandedText, partitionColumns, false); + } + + public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, Map<String, String> properties, + List<String> partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, + String expandedText, List<FieldSchema> partitionColumns, boolean icebergNativeView) { super(viewName, schema, originalText, expandedText); this.comment = comment; this.properties = properties; @@ -54,6 +65,7 @@ public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, this.ifNotExists = ifNotExists; this.replace = replace; this.partitionColumns = partitionColumns; + this.icebergNativeView = icebergNativeView; } @Explain(displayName = "partition columns") @@ -89,6 +101,11 @@ public boolean isReplace() { return replace; } + @Explain(displayName = "iceberg native view", displayOnlyOnTrue = true) + public boolean isIcebergNativeView() { + return icebergNativeView; + } + /** * @param replicationSpec Sets the replication spec governing this create. * This parameter will have meaningful values only for creates happening as a result of a replication. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index b6a41a8d4fb..f8eed31cf1e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.StorageFormat; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Map; /** @@ -46,6 +48,10 @@ public CreateViewOperation(DDLOperationContext context, CreateViewDesc desc) { @Override public int execute() throws HiveException { + if (desc.isIcebergNativeView()) { + return executeIcebergNativeView(); + } + Table oldview = context.getDb().getTable(desc.getViewName(), false); if (oldview != null) { boolean isReplace = desc.isReplace(); @@ -82,6 +88,11 @@ public int execute() throws HiveException { if (desc.getProperties() != null) { oldview.getTTable().getParameters().putAll(desc.getProperties()); } + if (desc.isIcebergNativeView()) { + oldview.setProperty(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY, "true"); + } else { + oldview.getParameters().remove(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY); + } oldview.setPartCols(desc.getPartitionColumns()); oldview.checkValidity(null); @@ -105,6 +116,118 @@ public int execute() throws HiveException { return 0; } + /** + * Creates a native Iceberg view via the configured default Iceberg catalog (HiveCatalog, REST + * catalog with {@link org.apache.iceberg.view.ViewCatalog}, etc.). + */ + private int executeIcebergNativeView() throws HiveException { + Table oldview = context.getDb().getTable(desc.getViewName(), false); + + if (oldview != null) { + boolean isReplace = desc.isReplace(); + + if (desc.getReplicationSpec().isInReplicationScope()) { + Map<String, String> dbParams = context.getDb().getDatabase(oldview.getDbName()).getParameters(); + if (desc.getReplicationSpec().allowEventReplacementInto(dbParams)) { + isReplace = true; + } else { + LOG.debug("DDLTask: Create Iceberg native view is skipped as view {} is newer than update", + desc.getViewName()); + return 0; + } + } + + if (!isReplace) { + if (desc.getIfNotExists()) { + return 0; + } + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName())); + } + } + + TableName name = HiveTableName.of(desc.getViewName()); + boolean replace = oldview != null || desc.isReplace(); + + try { + boolean created = invokeNativeIcebergViewSupport( + name.getDb(), + name.getTable(), + replace, + desc.getIfNotExists()); + if (!created) { + return 0; + } + } catch (HiveException e) { + throw e; + } catch (Exception e) { + throw new HiveException(e); + } + + Table viewTbl = context.getDb().getTable(desc.getViewName(), false); + if (viewTbl != null) { + DDLUtils.addIfAbsentByName(new WriteEntity(viewTbl, WriteEntity.WriteType.DDL_NO_LOCK), + context.getWork().getOutputs()); + DataContainer dc = new DataContainer(viewTbl.getTTable()); + context.getQueryState().getLineageState().setLineage(new Path(desc.getViewName()), dc, viewTbl.getCols()); + } + return 0; + } + + /** + * Delegates to {@code org.apache.iceberg.hive.NativeIcebergViewSupport} when the Iceberg catalog + * module is on the classpath (avoids a Maven reactor cycle between {@code hive-exec} and + * {@code hive-iceberg-catalog}). + */ + private boolean invokeNativeIcebergViewSupport( + String databaseName, String viewName, boolean replace, boolean ifNotExists) + throws Exception { + Class<?> supportClass; + Method method; + try { + supportClass = Class.forName("org.apache.iceberg.hive.NativeIcebergViewSupport"); + method = + supportClass.getMethod( + "createOrReplaceNativeView", + org.apache.hadoop.conf.Configuration.class, + String.class, + String.class, + java.util.List.class, + String.class, + Map.class, + String.class, + boolean.class, + boolean.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + throw new HiveException( + "Native Iceberg views require hive-iceberg-catalog on the classpath " + + "(org.apache.iceberg.hive.NativeIcebergViewSupport is missing).", + e); + } + try { + return (Boolean) + method.invoke( + null, + context.getConf(), + databaseName, + viewName, + desc.getSchema(), + desc.getExpandedText(), + desc.getProperties(), + desc.getComment(), + replace, + ifNotExists); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + if (cause.getClass().getName().endsWith("AlreadyExistsException")) { + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName()), cause); + } + if (cause instanceof Exception) { + throw (Exception) cause; + } + throw new HiveException(cause); + } + } + private Table createViewObject() throws HiveException { TableName name = HiveTableName.of(desc.getViewName()); Table view = new Table(name.getDb(), name.getTable()); @@ -121,6 +244,9 @@ private Table createViewObject() throws HiveException { if (desc.getProperties() != null) { view.getParameters().putAll(desc.getProperties()); } + if (desc.isIcebergNativeView()) { + view.setProperty(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY, "true"); + } if (!CollectionUtils.isEmpty(desc.getPartitionColumns())) { view.setPartCols(desc.getPartitionColumns());
