This is an automated email from the ASF dual-hosted git repository. difin pushed a commit to branch iceberg_native_views in repository https://gitbox.apache.org/repos/asf/hive.git
commit 8dacc958369e53a10aeda498b48f57b498dc7dbd Author: Dmitriy Fingerman <[email protected]> AuthorDate: Tue Apr 28 13:06:24 2026 -0400 fixes --- .../iceberg/hive/NativeIcebergViewSupport.java | 40 +++++++- .../iceberg/hive/TestNativeIcebergViewSupport.java | 5 +- .../iceberg/mr/hive/HiveIcebergStorageHandler.java | 31 ++++++ .../positive/llap/iceberg_native_view.q.out | 2 - .../llap/iceberg_rest_catalog_gravitino.q.out | 4 - .../positive/llap/iceberg_rest_catalog_hms.q.out | 4 - .../ql/ddl/view/create/CreateViewAnalyzer.java | 50 ++++++--- .../hive/ql/ddl/view/create/CreateViewDesc.java | 26 +++-- .../ql/ddl/view/create/CreateViewOperation.java | 113 ++++++++------------- .../hive/ql/metadata/CreateNativeViewRequest.java | 92 +++++++++++++++++ .../hive/ql/metadata/HiveStorageHandler.java | 48 +++++++++ .../apache/hadoop/hive/ql/parse/StorageFormat.java | 17 ++++ 12 files changed, 318 insertions(+), 114 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java index ace4ea703b8..a96037d9c29 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java @@ -30,6 +30,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.view.ViewBuilder; /** @@ -38,8 +39,22 @@ */ public final class NativeIcebergViewSupport { - /** HMS parameter aligned with Hive's {@code CreateViewDesc#ICEBERG_NATIVE_VIEW_PROPERTY}. */ - public static final String ICEBERG_NATIVE_VIEW_PROPERTY = "hive.iceberg.native.view"; + /** + * HMS table parameter recording the storage handler FQCN for native views; must match the constant + * {@code NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM} on {@code org.apache.hadoop.hive.ql.metadata.HiveStorageHandler}. + */ + public static final String NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM = "hive.storage.native.view.handler"; + + /** Value stored with {@link #NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM} for Iceberg native views. */ + public static final String NATIVE_ICEBERG_VIEW_HANDLER_FQCN = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; + + /** + * HMS / Iceberg view marker entries for a native Iceberg catalog view (same map as + * {@code HiveIcebergStorageHandler#getNativeViewHmsTableProperties()}). + */ + public static Map<String, String> defaultNativeViewMarkerTableProperties() { + return Map.of(NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM, NATIVE_ICEBERG_VIEW_HANDLER_FQCN); + } private NativeIcebergViewSupport() { } @@ -64,7 +79,7 @@ public static boolean createOrReplaceNativeView(Configuration conf, String datab } ViewBuilder builder = startViewBuilder(viewCatalog, identifier, fieldSchemas, viewSql); - builder = applyCommentAndTblProps(builder, tblProperties, comment); + builder = applyCommentAndTblProps(builder, mergeDefaultNativeViewTableProperties(tblProperties), comment); commitView(builder, replace); return true; } finally { @@ -94,8 +109,23 @@ private static ViewBuilder startViewBuilder( .buildView(identifier) .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) - .withQuery("hive", viewSql) - .withProperty(ICEBERG_NATIVE_VIEW_PROPERTY, "true"); + .withQuery("hive", viewSql); + } + + /** + * Fills Iceberg native-view HMS / view marker properties when absent (e.g. direct catalog callers). + * Handlers that delegate here after {@code HiveStorageHandler#getNativeViewHmsTableProperties()} already + * supplied markers get the same result. + */ + public static Map<String, String> mergeDefaultNativeViewTableProperties(Map<String, String> tblProperties) { + Map<String, String> merged = Maps.newHashMap(); + if (tblProperties != null) { + merged.putAll(tblProperties); + } + for (Map.Entry<String, String> e : defaultNativeViewMarkerTableProperties().entrySet()) { + merged.putIfAbsent(e.getKey(), e.getValue()); + } + return merged; } private static ViewBuilder applyCommentAndTblProps( diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java index daf187b5c09..72829eb0811 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java @@ -96,8 +96,9 @@ public void testCreateCommitsNativeViewWithMarkerProperty() throws Exception { TableIdentifier id = TableIdentifier.of(DB, VIEW); assertThat(cat.viewExists(id)).isTrue(); View view = cat.loadView(id); - assertThat(view.properties().get(NativeIcebergViewSupport.ICEBERG_NATIVE_VIEW_PROPERTY)) - .isEqualTo("true"); + assertThat( + view.properties().get(NativeIcebergViewSupport.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM)) + .isEqualTo(NativeIcebergViewSupport.NATIVE_ICEBERG_VIEW_HANDLER_FQCN); assertThat(view.properties().get("comment")).isEqualTo("hello-view"); assertThat(view.properties().get("k1")).isEqualTo("v1"); HiveViewOperations ops = (HiveViewOperations) ((BaseView) view).operations(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 65852f1a855..06a0a97c4e4 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -88,6 +88,7 @@ import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.metadata.CreateNativeViewRequest; import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -183,6 +184,7 @@ import org.apache.iceberg.hive.HiveTableOperations; import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.hive.MetastoreUtil; +import org.apache.iceberg.hive.NativeIcebergViewSupport; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; @@ -383,6 +385,35 @@ public boolean supportsPartitioning() { return true; } + @Override + public boolean supportsNativeViewCatalog() { + return true; + } + + @Override + public Map<String, String> getNativeViewHmsTableProperties() { + return NativeIcebergViewSupport.defaultNativeViewMarkerTableProperties(); + } + + @Override + public boolean createOrReplaceNativeView(Configuration conf, CreateNativeViewRequest request) + throws HiveException { + try { + return NativeIcebergViewSupport.createOrReplaceNativeView( + conf, + request.getDatabaseName(), + request.getViewName(), + request.getSchema(), + request.getExpandedText(), + request.getProperties(), + request.getComment(), + request.isReplace(), + request.isIfNotExists()); + } catch (Exception e) { + throw new HiveException(e); + } + } + /** * @param jobConf Job configuration for InputFormat to access * @param deserializer Deserializer diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out index 449f9f39f66..50bf9ae6bb5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out @@ -207,7 +207,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW #### A masked pattern was here #### @@ -278,7 +277,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW #### A masked pattern was here #### diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 3e50a68ef90..f1e578617dc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -225,7 +225,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -299,7 +298,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -370,7 +368,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -444,7 +441,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 3e50a68ef90..f1e578617dc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -225,7 +225,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -299,7 +298,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -370,7 +368,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest @@ -444,7 +441,6 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} - hive.iceberg.native.view true #### A masked pattern was here #### table_type ICEBERG-VIEW type rest diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index d750801172a..fdfdb469900 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -42,13 +43,13 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.StorageFormat; /** * Analyzer for create view commands. */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { - private static final String ICEBERG_STORAGE_HANDLER_CLASS = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); @@ -79,7 +80,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; ASTNode viewMetadataFormat = children.remove(HiveParser.TOK_VIEWMETADATAFORMAT); - boolean icebergNativeView = resolveNativeIcebergView(viewMetadataFormat); + String nativeViewStorageHandlerClass = resolveNativeViewStorageHandlerClass(viewMetadataFormat); assert children.isEmpty(); @@ -100,7 +101,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List<FieldSchema> partitionColumns = getPartitionColumns(partitionColumnNames); setColumnAccessInfo(analyzer.getColumnAccessInfo()); CreateViewDesc desc = new CreateViewDesc(fqViewName, schema, comment, properties, partitionColumnNames, - ifNotExists, orReplace, originalText, expandedText, partitionColumns, icebergNativeView); + ifNotExists, orReplace, originalText, expandedText, partitionColumns, nativeViewStorageHandlerClass); validateCreateView(desc, analyzer); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); @@ -198,26 +199,45 @@ private List<FieldSchema> getPartitionColumns(List<String> partitionColumnNames) } /** - * Native Iceberg view when {@code STORED BY iceberg} is present, or when it is omitted and - * {@code hive.default.storage.handler.class} is the Iceberg storage handler class. + * Returns the FQCN of the storage handler that should own native-catalog view metadata, or {@code null} for a + * classic HMS virtual view. Uses {@link HiveStorageHandler#supportsNativeViewCatalog()} on the resolved handler. */ - private boolean resolveNativeIcebergView(ASTNode viewMetadataFormat) throws SemanticException { + private String resolveNativeViewStorageHandlerClass(ASTNode viewMetadataFormat) throws SemanticException { + String handlerClass; if (viewMetadataFormat != null) { if (viewMetadataFormat.getChildCount() != 1) { throw new SemanticException("Internal error: expected single handler identifier in view metadata"); } - String handler = ((ASTNode) viewMetadataFormat.getChild(0)).getText(); - if (handler == null || !handler.equalsIgnoreCase("iceberg")) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( - "Only STORED BY ICEBERG is supported for native views, got: " + handler)); + String identifier = ((ASTNode) viewMetadataFormat.getChild(0)).getText(); + handlerClass = StorageFormat.resolveStorageHandlerClassNameForView(identifier); + } else { + handlerClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); + if (handlerClass != null) { + handlerClass = handlerClass.trim(); + if (handlerClass.isEmpty()) { + handlerClass = null; + } + } + } + if (handlerClass == null || handlerClass.isEmpty()) { + return null; + } + try { + HiveStorageHandler handler = HiveUtils.getStorageHandler(conf, handlerClass); + if (handler != null && handler.supportsNativeViewCatalog()) { + return handlerClass; + } + } catch (HiveException e) { + if (viewMetadataFormat != null) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg(e.getMessage()), e); } - return true; + return null; } - String defaultHandler = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); - if (defaultHandler == null || defaultHandler.isEmpty()) { - return false; + if (viewMetadataFormat != null) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( + "Native view metadata is not supported for storage handler: " + handlerClass)); } - return ICEBERG_STORAGE_HANDLER_CLASS.equals(defaultHandler.trim()); + return null; } private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index bbdf0df0da2..3210d3d988d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -34,16 +34,14 @@ public class CreateViewDesc extends AbstractCreateViewDesc { private static final long serialVersionUID = 1L; - /** HMS table property set when the view is declared with {@code STORED BY ICEBERG} (native Iceberg view). */ - public static final String ICEBERG_NATIVE_VIEW_PROPERTY = "hive.iceberg.native.view"; - private final String comment; private final Map<String, String> properties; private final List<String> partitionColumnNames; private final boolean ifNotExists; private final boolean replace; private final List<FieldSchema> partitionColumns; - private final boolean icebergNativeView; + /** FQCN of storage handler for native-catalog view, or {@code null} for a classic HMS virtual view. */ + private final String nativeViewStorageHandlerClass; private ReplicationSpec replicationSpec = null; private String ownerName = null; @@ -52,12 +50,12 @@ public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, List<String> partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, String expandedText, List<FieldSchema> partitionColumns) { this(viewName, schema, comment, properties, partitionColumnNames, ifNotExists, replace, originalText, - expandedText, partitionColumns, false); + expandedText, partitionColumns, null); } public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, Map<String, String> properties, List<String> partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, - String expandedText, List<FieldSchema> partitionColumns, boolean icebergNativeView) { + String expandedText, List<FieldSchema> partitionColumns, String nativeViewStorageHandlerClass) { super(viewName, schema, originalText, expandedText); this.comment = comment; this.properties = properties; @@ -65,7 +63,7 @@ public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, this.ifNotExists = ifNotExists; this.replace = replace; this.partitionColumns = partitionColumns; - this.icebergNativeView = icebergNativeView; + this.nativeViewStorageHandlerClass = nativeViewStorageHandlerClass; } @Explain(displayName = "partition columns") @@ -101,9 +99,17 @@ public boolean isReplace() { return replace; } - @Explain(displayName = "iceberg native view", displayOnlyOnTrue = true) - public boolean isIcebergNativeView() { - return icebergNativeView; + /** + * @return FQCN of the {@link HiveStorageHandler} that stores view metadata in an external catalog, or + * {@code null} for a classic HMS-only virtual view. + */ + @Explain(displayName = "native view storage handler", displayOnlyOnTrue = true) + public String getNativeViewStorageHandlerClass() { + return nativeViewStorageHandlerClass; + } + + public boolean usesNativeViewCatalog() { + return nativeViewStorageHandlerClass != null && !nativeViewStorageHandlerClass.trim().isEmpty(); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index f8eed31cf1e..c4b6d59d2c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -29,13 +29,15 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; +import org.apache.hadoop.hive.ql.metadata.CreateNativeViewRequest; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.StorageFormat; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.util.HashMap; import java.util.Map; /** @@ -48,8 +50,8 @@ public CreateViewOperation(DDLOperationContext context, CreateViewDesc desc) { @Override public int execute() throws HiveException { - if (desc.isIcebergNativeView()) { - return executeIcebergNativeView(); + if (desc.usesNativeViewCatalog()) { + return executeNativeCatalogView(); } Table oldview = context.getDb().getTable(desc.getViewName(), false); @@ -88,11 +90,7 @@ public int execute() throws HiveException { if (desc.getProperties() != null) { oldview.getTTable().getParameters().putAll(desc.getProperties()); } - if (desc.isIcebergNativeView()) { - oldview.setProperty(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY, "true"); - } else { - oldview.getParameters().remove(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY); - } + clearNativeViewCatalogMarkersIfPresent(oldview); oldview.setPartCols(desc.getPartitionColumns()); oldview.checkValidity(null); @@ -117,10 +115,9 @@ public int execute() throws HiveException { } /** - * Creates a native Iceberg view via the configured default Iceberg catalog (HiveCatalog, REST - * catalog with {@link org.apache.iceberg.view.ViewCatalog}, etc.). + * Creates a native-catalog view via the {@link HiveStorageHandler} selected at compile time. */ - private int executeIcebergNativeView() throws HiveException { + private int executeNativeCatalogView() throws HiveException { Table oldview = context.getDb().getTable(desc.getViewName(), false); if (oldview != null) { @@ -131,7 +128,7 @@ private int executeIcebergNativeView() throws HiveException { if (desc.getReplicationSpec().allowEventReplacementInto(dbParams)) { isReplace = true; } else { - LOG.debug("DDLTask: Create Iceberg native view is skipped as view {} is newer than update", + LOG.debug("DDLTask: Create native-catalog view is skipped as view {} is newer than update", desc.getViewName()); return 0; } @@ -149,15 +146,32 @@ private int executeIcebergNativeView() throws HiveException { boolean replace = oldview != null || desc.isReplace(); try { - boolean created = invokeNativeIcebergViewSupport( - name.getDb(), - name.getTable(), - replace, - desc.getIfNotExists()); + HiveStorageHandler handler = + HiveUtils.getStorageHandler(context.getConf(), desc.getNativeViewStorageHandlerClass()); + Map<String, String> viewProperties = new HashMap<>(); + if (desc.getProperties() != null) { + viewProperties.putAll(desc.getProperties()); + } + viewProperties.putAll(handler.getNativeViewHmsTableProperties()); + CreateNativeViewRequest request = + new CreateNativeViewRequest( + name.getDb(), + name.getTable(), + desc.getSchema(), + desc.getExpandedText(), + viewProperties, + desc.getComment(), + replace, + desc.getIfNotExists()); + boolean created = handler.createOrReplaceNativeView(context.getConf(), request); if (!created) { return 0; } } catch (HiveException e) { + Throwable cause = e.getCause(); + if (cause != null && cause.getClass().getName().endsWith("AlreadyExistsException")) { + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName()), cause); + } throw e; } catch (Exception e) { throw new HiveException(e); @@ -173,59 +187,17 @@ private int executeIcebergNativeView() throws HiveException { return 0; } - /** - * Delegates to {@code org.apache.iceberg.hive.NativeIcebergViewSupport} when the Iceberg catalog - * module is on the classpath (avoids a Maven reactor cycle between {@code hive-exec} and - * {@code hive-iceberg-catalog}). - */ - private boolean invokeNativeIcebergViewSupport( - String databaseName, String viewName, boolean replace, boolean ifNotExists) - throws Exception { - Class<?> supportClass; - Method method; - try { - supportClass = Class.forName("org.apache.iceberg.hive.NativeIcebergViewSupport"); - method = - supportClass.getMethod( - "createOrReplaceNativeView", - org.apache.hadoop.conf.Configuration.class, - String.class, - String.class, - java.util.List.class, - String.class, - Map.class, - String.class, - boolean.class, - boolean.class); - } catch (ClassNotFoundException | NoSuchMethodException e) { - throw new HiveException( - "Native Iceberg views require hive-iceberg-catalog on the classpath " - + "(org.apache.iceberg.hive.NativeIcebergViewSupport is missing).", - e); + private void clearNativeViewCatalogMarkersIfPresent(Table oldview) throws HiveException { + Map<String, String> params = oldview.getParameters(); + if (params == null) { + return; } - try { - return (Boolean) - method.invoke( - null, - context.getConf(), - databaseName, - viewName, - desc.getSchema(), - desc.getExpandedText(), - desc.getProperties(), - desc.getComment(), - replace, - ifNotExists); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause() == null ? e : e.getCause(); - if (cause.getClass().getName().endsWith("AlreadyExistsException")) { - throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName()), cause); - } - if (cause instanceof Exception) { - throw (Exception) cause; - } - throw new HiveException(cause); + String fqcn = params.get(HiveStorageHandler.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM); + if (fqcn == null) { + return; } + HiveStorageHandler handler = HiveUtils.getStorageHandler(context.getConf(), fqcn); + handler.clearNativeViewHmsTableProperties(params); } private Table createViewObject() throws HiveException { @@ -244,9 +216,6 @@ private Table createViewObject() throws HiveException { if (desc.getProperties() != null) { view.getParameters().putAll(desc.getProperties()); } - if (desc.isIcebergNativeView()) { - view.setProperty(CreateViewDesc.ICEBERG_NATIVE_VIEW_PROPERTY, "true"); - } if (!CollectionUtils.isEmpty(desc.getPartitionColumns())) { view.setPartCols(desc.getPartitionColumns()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java new file mode 100644 index 00000000000..f45f697ecca --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * Parameters for {@link HiveStorageHandler#createOrReplaceNativeView} (native catalog view DDL). + */ +public final class CreateNativeViewRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String viewName; + private final List<FieldSchema> schema; + private final String expandedText; + private final Map<String, String> properties; + private final String comment; + private final boolean replace; + private final boolean ifNotExists; + + public CreateNativeViewRequest( + String databaseName, + String viewName, + List<FieldSchema> schema, + String expandedText, + Map<String, String> properties, + String comment, + boolean replace, + boolean ifNotExists) { + this.databaseName = databaseName; + this.viewName = viewName; + this.schema = schema; + this.expandedText = expandedText; + this.properties = properties == null ? null : Collections.unmodifiableMap(properties); + this.comment = comment; + this.replace = replace; + this.ifNotExists = ifNotExists; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getViewName() { + return viewName; + } + + public List<FieldSchema> getSchema() { + return schema; + } + + public String getExpandedText() { + return expandedText; + } + + public Map<String, String> getProperties() { + return properties; + } + + public String getComment() { + return comment; + } + + public boolean isReplace() { + return replace; + } + + public boolean isIfNotExists() { + return ifNotExists; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 520c52a24a8..563ef5c35a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService; import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -1028,6 +1029,53 @@ default void setMergeTaskDeleteProperties(TableDesc tableDesc) { throw new UnsupportedOperationException("Storage handler does not support getting custom delete merge schema."); } + /** + * HMS table parameter recording the FQCN of the {@link HiveStorageHandler} that created a native-catalog view, + * used when altering back to a classic view to clear handler-specific HMS markers. + */ + String NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM = "hive.storage.native.view.handler"; + + /** + * @return {@code true} if this handler may store CREATE VIEW text and column metadata in an external catalog + * (native view) rather than only as a classic HMS virtual view. + */ + default boolean supportsNativeViewCatalog() { + return false; + } + + /** + * HMS table-level parameters to set on the metastore stub for a native-catalog view (markers, etc.). + * Keys should be removed when {@linkplain #clearNativeViewHmsTableProperties(Map)} is invoked for the same + * handler class recorded under {@link #NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM}. + */ + default Map<String, String> getNativeViewHmsTableProperties() { + return Collections.emptyMap(); + } + + /** + * Removes entries added for native-catalog views (see {@link #getNativeViewHmsTableProperties()}). + */ + default void clearNativeViewHmsTableProperties(Map<String, String> tableParameters) { + if (tableParameters == null) { + return; + } + for (String k : getNativeViewHmsTableProperties().keySet()) { + tableParameters.remove(k); + } + tableParameters.remove(NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM); + } + + /** + * Creates or replaces a view in the external catalog backing this handler. + * + * @return {@code false} if the operation was skipped (e.g. IF NOT EXISTS and the view already exists) + */ + default boolean createOrReplaceNativeView(Configuration conf, CreateNativeViewRequest request) + throws HiveException { + throw new HiveException( + "createOrReplaceNativeView is not supported by storage handler " + getClass().getName()); + } + default boolean supportsDefaultColumnValues(Map<String, String> tblProps) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 2472ad44ad0..4f69e1173f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -89,6 +89,23 @@ public String outputFormat() { } } + /** + * Resolves {@code STORED BY <identifier>} for CREATE VIEW (short names such as {@code ICEBERG} or an FQCN). + */ + public static String resolveStorageHandlerClassNameForView(String identifierText) throws SemanticException { + if (StringUtils.isBlank(identifierText)) { + throw new SemanticException("Storage handler identifier in CREATE VIEW cannot be empty"); + } + String text = identifierText.trim(); + for (StorageHandlerTypes type : StorageHandlerTypes.NON_DEFAULT_TYPES) { + if (type.name().equalsIgnoreCase(text)) { + Objects.requireNonNull(type.className()); + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(type.className())); + } + } + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(text)); + } + public StorageFormat(Configuration conf) { this.conf = conf; this.serdeProps = new HashMap<String, String>();
