This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a commit to branch branch-4.0 in repository https://gitbox.apache.org/repos/asf/hive.git
commit 9cbacbfbff1e55954e86d9e87b97d47c0a47a550 Author: Krisztian Kasa <[email protected]> AuthorDate: Mon Mar 11 13:55:47 2024 +0100 HIVE-28006: Materialized view with aggregate function incorrectly shows it allows incremental rebuild (Krisztian Kasa, reviewed by Aman Sinha, Laszlo Bodor) --- .../positive/show_iceberg_materialized_views.q.out | 60 ++-- .../AlterMaterializedViewRebuildAnalyzer.java | 33 +- .../show/ShowMaterializedViewsFormatter.java | 28 +- .../ql/metadata/HiveMaterializedViewsRegistry.java | 13 +- .../ql/metadata/HiveRelOptMaterialization.java | 8 +- .../rules/views/IncrementalRebuildMode.java | 34 ++ ...rializedViewIncrementalRewritingRelVisitor.java | 276 ++++++++++++---- .../views/MaterializedViewRewritingRelVisitor.java | 69 ++-- .../ql/metadata/TestMaterializedViewsCache.java | 3 +- ...TestHiveAugmentSnapshotMaterializationRule.java | 18 +- .../views/TestHivePushdownSnapshotFilterRule.java | 4 +- ...rializedViewIncrementalRewritingRelVisitor.java | 364 +++++++++++++++++++++ .../calcite/rules/views/TestRuleBase.java | 58 +++- .../clientpositive/show_materialized_views.q | 3 + .../clientpositive/llap/show_json_format.q.out | 2 +- .../llap/show_materialized_views.q.out | 72 ++-- 16 files changed, 805 insertions(+), 240 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out index 9b784e9d54d..8aefc76e6f9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_iceberg_materialized_views.q.out @@ -224,12 +224,12 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only -shtb_aggr_view2 Yes Manual refresh Available in presence of insert operations only -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available in presence of insert operations only -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only -shtb_view_on_native Yes Manual refresh Available +shtb_aggr_view1 Yes Manual refresh Available for insert operations only +shtb_aggr_view2 Yes Manual refresh Available for insert operations only +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only +shtb_view_on_native Yes Manual refresh Available for insert operations only PREHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS '%test%' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS '%test%' @@ -245,16 +245,16 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS '%test%' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS '%view2' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS '%view2' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view2 Yes Manual refresh Available in presence of insert operations only -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available in presence of insert operations only -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only +shtb_aggr_view2 Yes Manual refresh Available for insert operations only +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: USE test2 PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:test2 @@ -266,8 +266,8 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS 'shtb_%' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test2_view2 No Manual refresh Unknown +shtb_test1_view1 No Manual refresh Not available +shtb_test2_view2 No Manual refresh Not available PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:default @@ -289,19 +289,19 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only -shtb_aggr_view2 Yes Manual refresh Available in presence of insert operations only -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available in presence of insert operations only -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only -shtb_view_on_native Yes Manual refresh Available +shtb_aggr_view1 Yes Manual refresh Available for insert operations only +shtb_aggr_view2 Yes Manual refresh Available for insert operations only +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only +shtb_view_on_native Yes Manual refresh Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test2_view2 No Manual refresh Unknown +shtb_test1_view1 No Manual refresh Not available +shtb_test2_view2 No Manual refresh Not available PREHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS IN test1 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS IN test1 @@ -317,12 +317,12 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only -shtb_aggr_view2 Yes Manual refresh Available in presence of insert operations only -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available in presence of insert operations only -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only -shtb_view_on_native Yes Manual refresh Available +shtb_aggr_view1 Yes Manual refresh Available for insert operations only +shtb_aggr_view2 Yes Manual refresh Available for insert operations only +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only +shtb_view_on_native Yes Manual refresh Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS IN default PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN default @@ -342,8 +342,8 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 "shtb_test%" POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available in presence of insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: DESCRIBE FORMATTED test1.shtb_full_view2 PREHOOK: type: DESCTABLE PREHOOK: Input: test1@shtb_full_view2 @@ -553,7 +553,7 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM `database` LIKE "fooview" POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -fooview Yes Manual refresh (Valid until source tables modified) Available in presence of insert operations only +fooview Yes Manual refresh (Valid until source tables modified) Available for insert operations only PREHOOK: query: DESCRIBE FORMATTED `database`.`fooview` PREHOOK: type: DESCTABLE PREHOOK: Input: database@fooview diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java index c0990e8c568..2c1f148d303 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java @@ -353,29 +353,32 @@ public class AlterMaterializedViewRebuildAnalyzer extends CalcitePlanner { MaterializedViewRewritingRelVisitor visitor = new MaterializedViewRewritingRelVisitor(acidView); visitor.go(basePlan); - if (visitor.isRewritingAllowed()) { - if (!materialization.isSourceTablesUpdateDeleteModified()) { - // Trigger rewriting to remove UNION branch with MV + switch (visitor.getIncrementalRebuildMode()) { + case INSERT_ONLY: + if (materialization.isSourceTablesUpdateDeleteModified()) { + return calcitePreMVRewritingPlan; + } + if (visitor.isContainsAggregate()) { return applyAggregateInsertIncremental(basePlan, mdProvider, executorProvider, optCluster, calcitePreMVRewritingPlan); } else { return applyJoinInsertIncremental(basePlan, mdProvider, executorProvider); } - } else { - // count(*) is necessary for determine which rows should be deleted from the view - // if view definition does not have it incremental rebuild can not be performed - if (acidView && visitor.isContainsAggregate() && visitor.getCountIndex() >= 0) { - return applyAggregateInsertDeleteIncremental(basePlan, mdProvider, executorProvider); + case AVAILABLE: + if (!materialization.isSourceTablesUpdateDeleteModified()) { + return applyAggregateInsertIncremental(basePlan, mdProvider, executorProvider, optCluster, calcitePreMVRewritingPlan); } else { + return applyAggregateInsertDeleteIncremental(basePlan, mdProvider, executorProvider); + } + case NOT_AVAILABLE: + default: + if (materialization.isSourceTablesUpdateDeleteModified()) { + // calcitePreMVRewritingPlan is already got the optimizations by applyPreJoinOrderingTransforms prior to calling + // applyMaterializedViewRewriting in CalcitePlanner.CalcitePlannerAction.apply return calcitePreMVRewritingPlan; + } else { + return applyPreJoinOrderingTransforms(basePlan, mdProvider, executorProvider); } - } - } else if (materialization.isSourceTablesUpdateDeleteModified()) { - // calcitePreMVRewritingPlan is already got the optimizations by applyPreJoinOrderingTransforms prior calling - // applyMaterializedViewRewriting in CalcitePlanner.CalcitePlannerAction.apply - return calcitePreMVRewritingPlan; - } else { - return applyPreJoinOrderingTransforms(basePlan, mdProvider, executorProvider); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java index c8c43af74c4..71f443c8dd4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java @@ -40,8 +40,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW; -import static org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization.IncrementalRebuildMode.UNKNOWN; import static org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm.ALL; +import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode.NOT_AVAILABLE; /** * Formats SHOW MATERIALIZED VIEWS results. @@ -136,27 +136,13 @@ abstract class ShowMaterializedViewsFormatter { @NotNull private static String formatIncrementalRebuildMode(Table materializedView) { - String incrementalRebuild; HiveRelOptMaterialization relOptMaterialization = HiveMaterializedViewsRegistry.get(). - getRewritingMaterializedView(materializedView.getDbName(), materializedView.getTableName(), ALL); - if (relOptMaterialization == null || relOptMaterialization.getRebuildMode() == UNKNOWN) { - incrementalRebuild = "Unknown"; - } else { - switch (relOptMaterialization.getRebuildMode()) { - case AVAILABLE: - incrementalRebuild = "Available"; - break; - case INSERT_ONLY: - incrementalRebuild = "Available in presence of insert operations only"; - break; - case NOT_AVAILABLE: - incrementalRebuild = "Not available"; - break; - default: - incrementalRebuild = "Unknown"; - break; - } + getRewritingMaterializedView(materializedView.getDbName(), materializedView.getTableName(), ALL); + + if (relOptMaterialization == null) { + return NOT_AVAILABLE.getMessage(); } - return incrementalRebuild; + + return relOptMaterialization.getRebuildMode().getMessage(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 9c5bdfe18af..32ffac3a03f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.MaterializedViewIncrementalRewritingRelVisitor; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -238,16 +239,8 @@ public final class HiveMaterializedViewsRegistry { determineIncrementalRebuildMode(plan.getPlan()), plan.getAst()); } - private HiveRelOptMaterialization.IncrementalRebuildMode determineIncrementalRebuildMode(RelNode definitionPlan) { - MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); - visitor.go(definitionPlan); - if (!visitor.hasAllowedOperatorsOnly()) { - return HiveRelOptMaterialization.IncrementalRebuildMode.NOT_AVAILABLE; - } - if (visitor.isContainsAggregate() && !visitor.hasCountStar() || visitor.isInsertAllowedOnly()) { - return HiveRelOptMaterialization.IncrementalRebuildMode.INSERT_ONLY; - } - return HiveRelOptMaterialization.IncrementalRebuildMode.AVAILABLE; + private IncrementalRebuildMode determineIncrementalRebuildMode(RelNode definitionPlan) { + return new MaterializedViewIncrementalRewritingRelVisitor().go(definitionPlan).getIncrementalRebuildMode(); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java index 04548bfb801..920db4f336a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java @@ -24,6 +24,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; import org.apache.hadoop.hive.metastore.api.Materialization; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode; import org.apache.hadoop.hive.ql.parse.ASTNode; import java.util.EnumSet; @@ -37,13 +38,6 @@ import static org.apache.commons.collections.CollectionUtils.intersection; */ public class HiveRelOptMaterialization extends RelOptMaterialization { - public enum IncrementalRebuildMode { - AVAILABLE, - INSERT_ONLY, - NOT_AVAILABLE, - UNKNOWN - } - private final Set<RewriteAlgorithm> scope; private final boolean sourceTablesUpdateDeleteModified; private final boolean sourceTablesCompacted; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/IncrementalRebuildMode.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/IncrementalRebuildMode.java new file mode 100644 index 00000000000..8947585535c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/IncrementalRebuildMode.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +public enum IncrementalRebuildMode { + AVAILABLE("Available"), + INSERT_ONLY("Available for insert operations only"), + NOT_AVAILABLE("Not available"); + + private final String message; + + IncrementalRebuildMode(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewIncrementalRewritingRelVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewIncrementalRewritingRelVisitor.java index ddba4a251c6..ddd4d924b3c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewIncrementalRewritingRelVisitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewIncrementalRewritingRelVisitor.java @@ -17,97 +17,255 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelVisitor; -import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.util.ReflectUtil; +import org.apache.calcite.util.ReflectiveVisitor; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode.AVAILABLE; +import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode.INSERT_ONLY; +import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode.NOT_AVAILABLE; /** * This class is a helper to check whether a materialized view rebuild * can be transformed from INSERT OVERWRITE to INSERT INTO. * * We are verifying that: - * 1) Plan only uses legal operators (i.e., Filter, Project, - * Join, and TableScan) - * 2) Whether the plane has aggregate - * 3) Whether the plane has an count(*) aggregate function call + * <ul> + * <li>Plan only uses legal operators (i.e., Filter, Project, Join, and TableScan)</li> + * <li>Whether the plan has aggregate</li> + * <li>Whether the plan has count(*) aggregate function call</li> + * <li>Check whether aggregate functions are supported</li> + * </ul> */ -public class MaterializedViewIncrementalRewritingRelVisitor extends RelVisitor { +public class MaterializedViewIncrementalRewritingRelVisitor implements ReflectiveVisitor { + + private static final Logger LOG = LoggerFactory.getLogger(MaterializedViewIncrementalRewritingRelVisitor.class); - private boolean containsAggregate; - private boolean hasAllowedOperatorsOnly; - private boolean hasCountStar; - private boolean insertAllowedOnly; + private final ReflectUtil.MethodDispatcher<Result> dispatcher; public MaterializedViewIncrementalRewritingRelVisitor() { - this.containsAggregate = false; - this.hasAllowedOperatorsOnly = true; - this.hasCountStar = false; - this.insertAllowedOnly = false; + this.dispatcher = ReflectUtil.createMethodDispatcher( + Result.class, this, "visit", RelNode.class); } - @Override - public void visit(RelNode node, int ordinal, RelNode parent) { - if (node instanceof Aggregate) { - this.containsAggregate = true; - check((Aggregate) node); - super.visit(node, ordinal, parent); - } else if ( - node instanceof Filter || - node instanceof Project || - node instanceof Join) { - super.visit(node, ordinal, parent); - } else if (node instanceof TableScan) { - HiveTableScan scan = (HiveTableScan) node; - RelOptHiveTable hiveTable = (RelOptHiveTable) scan.getTable(); - if (hiveTable.getHiveTableMD().getStorageHandler() != null && - hiveTable.getHiveTableMD().getStorageHandler().areSnapshotsSupported()) { + /** + * Starts an iteration. + */ + public Result go(RelNode relNode) { + Result result = dispatcher.invoke(relNode); + if (result.containsAggregate) { + return result; + } + + if (result.incrementalRebuildMode == AVAILABLE) { + // Incremental rebuild of non-aggregate MV is not supported when any source table has delete operations. + return new Result(INSERT_ONLY); + } + + return result; + } + + public Result visit(RelNode relNode) { + // Only TS, Filter, Join, Project and Aggregate are supported + LOG.debug("Plan has unsupported operator {}", relNode); + return new Result(NOT_AVAILABLE); + } + + private Result visitChildOf(RelNode rel) { + return visitChildOf(rel, 0); + } + + private Result visitChildOf(RelNode rel, int index) { + return dispatcher.invoke(rel.getInput(index)); + } + + public Result visit(HiveTableScan scan) { + RelOptHiveTable hiveTable = (RelOptHiveTable) scan.getTable(); + + Table hiveTableMD = hiveTable.getHiveTableMD(); + if (hiveTableMD.getStorageHandler() != null) { + if (hiveTableMD.getStorageHandler().areSnapshotsSupported()) { // Incremental rebuild of materialized views with non-native source tables are not implemented // when any of the source tables has delete/update operation since the last rebuild - insertAllowedOnly = true; + LOG.debug("Table scan of non-native table {} with {} storage handler supports insert only materialized view" + + " incremental rebuild.", + hiveTableMD.getTableName(), hiveTableMD.getStorageHandler().getClass().getSimpleName()); + return new Result(INSERT_ONLY); + } else { + LOG.debug("Unsupported table type: non-native table {} with storage handler {}", + hiveTableMD.getTableName(), hiveTableMD.getStorageHandler().getClass().getSimpleName()); + return new Result(NOT_AVAILABLE); } - } else { - hasAllowedOperatorsOnly = false; } + + return new Result(AVAILABLE); + } + + public Result visit(HiveProject project) { + return visitChildOf(project); + } + + public Result visit(HiveFilter filter) { + return visitChildOf(filter); + } + + public Result visit(HiveJoin join) { + if (join.getJoinType() != JoinRelType.INNER) { + LOG.debug("Unsupported join type {}", join.getJoinType()); + return new Result(NOT_AVAILABLE); + } + + Result leftResult = visitChildOf(join, 0); + Result rightResult = visitChildOf(join, 1); + + boolean containsAggregate = leftResult.containsAggregate || rightResult.containsAggregate; + if (leftResult.incrementalRebuildMode == NOT_AVAILABLE || rightResult.incrementalRebuildMode == NOT_AVAILABLE) { + return new Result(NOT_AVAILABLE, containsAggregate); + } + if (leftResult.incrementalRebuildMode == INSERT_ONLY || rightResult.incrementalRebuildMode == INSERT_ONLY) { + return new Result(INSERT_ONLY, containsAggregate); + } + return new Result(AVAILABLE, containsAggregate); } - private void check(Aggregate aggregate) { + public Result visit(HiveAggregate aggregate) { + Result result = visitChildOf(aggregate); + if (result.incrementalRebuildMode == NOT_AVAILABLE) { + return new Result(result.incrementalRebuildMode, true, -1); + } + + Map<Integer, Set<SqlKind>> columnRefByAggregateCall = new HashMap<>(aggregate.getRowType().getFieldCount()); + + int countStarIndex = -1; for (int i = 0; i < aggregate.getAggCallList().size(); ++i) { AggregateCall aggregateCall = aggregate.getAggCallList().get(i); - if (aggregateCall.getAggregation().getKind() == SqlKind.COUNT && aggregateCall.getArgList().size() == 0) { - hasCountStar = true; - break; + if (aggregateCall.getAggregation().getKind() == SqlKind.COUNT && + aggregateCall.getArgList().isEmpty() && + !aggregateCall.isDistinct() && + !aggregateCall.isApproximate()) { + countStarIndex = i; + continue; + } + + for (Integer argIndex : aggregateCall.getArgList()) { + columnRefByAggregateCall.computeIfAbsent(argIndex, integer -> new HashSet<>()); + Set<SqlKind> aggregates = columnRefByAggregateCall.get(argIndex); + aggregates.add(aggregateCall.getAggregation().getKind()); } } - } - /** - * Starts an iteration. - */ - public RelNode go(RelNode p) { - visit(p, 0, null); - return p; - } + IncrementalRebuildMode incrementalRebuildMode = + result.incrementalRebuildMode == INSERT_ONLY || countStarIndex == -1 ? INSERT_ONLY : AVAILABLE; + LOG.debug("Initial incremental rebuild mode {} input's incremental rebuild mode {} count star index {}", + incrementalRebuildMode, result.incrementalRebuildMode, countStarIndex); - public boolean isContainsAggregate() { - return containsAggregate; - } + incrementalRebuildMode = updateBasedOnAggregates(aggregate, columnRefByAggregateCall, incrementalRebuildMode); - public boolean hasAllowedOperatorsOnly() { - return hasAllowedOperatorsOnly; + return new Result(incrementalRebuildMode, true, countStarIndex); } - public boolean isInsertAllowedOnly() { - return insertAllowedOnly; + private IncrementalRebuildMode updateBasedOnAggregates( + HiveAggregate aggregate, + Map<Integer, Set<SqlKind>> columnRefByAggregateCall, + IncrementalRebuildMode incrementalRebuildMode) { + + for (int i = 0; i < aggregate.getAggCallList().size(); ++i) { + AggregateCall aggregateCall = aggregate.getAggCallList().get(i); + switch (aggregateCall.getAggregation().getKind()) { + case COUNT: + if (aggregateCall.isDistinct() || aggregateCall.isApproximate()) { + LOG.debug("Unsupported aggregate function COUNT with distinct {} or approximate {}", + aggregateCall.isDistinct(), aggregateCall.isApproximate()); + return NOT_AVAILABLE; + } + + case SUM: + case SUM0: + break; + + case AVG: + Set<SqlKind> aggregates = columnRefByAggregateCall.get(aggregateCall.getArgList().get(0)); + if (!(aggregates.contains(SqlKind.SUM) && aggregates.contains(SqlKind.COUNT))) { + // We don't check if the Count is distinct or approximate here since these are not supported currently + // see count handling + LOG.debug("Unsupported aggregate function AVG: missing SUM and COUNT of the same column."); + return NOT_AVAILABLE; + } + break; + + case MIN: + case MAX: + incrementalRebuildMode = INSERT_ONLY; + LOG.debug("Found {} aggregate function. Incremental materialized view rebuild is supported in " + + "the presence of insert operations only", aggregateCall.getAggregation().getKind()); + break; + + default: + LOG.debug("Unsupported aggregate function {}.", aggregateCall.getAggregation().getKind()); + return NOT_AVAILABLE; + } + } + + return incrementalRebuildMode; } - public boolean hasCountStar() { - return hasCountStar; + public static final class Result { + private final IncrementalRebuildMode incrementalRebuildMode; + private final boolean containsAggregate; + private final int countStarIndex; + + private Result(IncrementalRebuildMode incrementalRebuildMode) { + this(incrementalRebuildMode, false, -1); + } + + private Result(IncrementalRebuildMode incrementalRebuildMode, boolean containsAggregate) { + this(incrementalRebuildMode, containsAggregate, -1); + } + + public Result( + IncrementalRebuildMode incrementalRebuildMode, + boolean containsAggregate, + int countStarIndex) { + this.incrementalRebuildMode = incrementalRebuildMode; + this.containsAggregate = containsAggregate; + this.countStarIndex = countStarIndex; + } + + public IncrementalRebuildMode getIncrementalRebuildMode() { + return incrementalRebuildMode; + } + + public boolean containsAggregate() { + return containsAggregate; + } + + public int getCountStarIndex() { + return countStarIndex; + } + + @Override + public String toString() { + return "Result{" + + "incrementalRebuildMode=" + incrementalRebuildMode + + ", containsAggregate=" + containsAggregate + + ", countStarIndex=" + countStarIndex + + '}'; + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java index d8ac6798221..b33467163ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java @@ -19,17 +19,11 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; -import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.ControlFlowException; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This class is a helper to check whether a materialized view rebuild @@ -45,18 +39,15 @@ import org.slf4j.LoggerFactory; */ public class MaterializedViewRewritingRelVisitor extends RelVisitor { - private static final Logger LOG = LoggerFactory.getLogger(MaterializedViewRewritingRelVisitor.class); - - private boolean containsAggregate; private final boolean fullAcidView; - private boolean rewritingAllowed; + private IncrementalRebuildMode incrementalRebuildMode; private int countIndex; public MaterializedViewRewritingRelVisitor(boolean fullAcidView) { this.containsAggregate = false; this.fullAcidView = fullAcidView; - this.rewritingAllowed = false; + this.incrementalRebuildMode = IncrementalRebuildMode.NOT_AVAILABLE; this.countIndex = -1; } @@ -77,42 +68,22 @@ public class MaterializedViewRewritingRelVisitor extends RelVisitor { // Project operator, we can continue super.visit(node, ordinal, parent); } - throw new ReturnedValue(false); + throw new ReturnedValue(IncrementalRebuildMode.NOT_AVAILABLE); } private void check(Union union) { // We found the Union if (union.getInputs().size() != 2) { // Bail out - throw new ReturnedValue(false); + throw new ReturnedValue(IncrementalRebuildMode.NOT_AVAILABLE); } // First branch should have the query (with write ID filter conditions) RelNode queryBranch = union.getInput(0); - new RelVisitor() { - @Override - public void visit(RelNode node, int ordinal, RelNode parent) { - if (node instanceof TableScan || - node instanceof Filter || - node instanceof Project || - node instanceof Join) { - // We can continue - super.visit(node, ordinal, parent); - } else if (node instanceof Aggregate && containsAggregate) { - Aggregate aggregate = (Aggregate) node; - for (int i = 0; i < aggregate.getAggCallList().size(); ++i) { - AggregateCall aggregateCall = aggregate.getAggCallList().get(i); - if (aggregateCall.getAggregation().getKind() == SqlKind.COUNT && aggregateCall.getArgList().size() == 0) { - countIndex = i + aggregate.getGroupCount(); - break; - } - } - // We can continue - super.visit(node, ordinal, parent); - } else { - throw new ReturnedValue(false); - } - } - }.go(queryBranch); + MaterializedViewIncrementalRewritingRelVisitor.Result result = + new MaterializedViewIncrementalRewritingRelVisitor().go(queryBranch); + incrementalRebuildMode = result.getIncrementalRebuildMode(); + containsAggregate = result.containsAggregate(); + countIndex = result.getCountStarIndex(); // Second branch should only have the MV new RelVisitor() { @@ -124,23 +95,23 @@ public class MaterializedViewRewritingRelVisitor extends RelVisitor { RelOptHiveTable hiveTable = (RelOptHiveTable) node.getTable(); if (!hiveTable.getHiveTableMD().isMaterializedView()) { // If it is not a materialized view, we do not rewrite it - throw new ReturnedValue(false); + throw new ReturnedValue(IncrementalRebuildMode.NOT_AVAILABLE); } if (containsAggregate && !fullAcidView) { // If it contains an aggregate and it is not a full acid table, // we do not rewrite it (we need MERGE support) - throw new ReturnedValue(false); + throw new ReturnedValue(IncrementalRebuildMode.NOT_AVAILABLE); } } else if (node instanceof Project) { // We can continue super.visit(node, ordinal, parent); } else { - throw new ReturnedValue(false); + throw new ReturnedValue(IncrementalRebuildMode.NOT_AVAILABLE); } } }.go(union.getInput(1)); // We pass all the checks, we can rewrite - throw new ReturnedValue(true); + throw new ReturnedValue(result.getIncrementalRebuildMode()); } /** @@ -151,7 +122,7 @@ public class MaterializedViewRewritingRelVisitor extends RelVisitor { visit(p, 0, null); } catch (ReturnedValue e) { // Rewriting cannot be performed - rewritingAllowed = e.value; + incrementalRebuildMode = e.incrementalRebuildMode; } return p; } @@ -160,8 +131,8 @@ public class MaterializedViewRewritingRelVisitor extends RelVisitor { return containsAggregate; } - public boolean isRewritingAllowed() { - return rewritingAllowed; + public IncrementalRebuildMode getIncrementalRebuildMode() { + return incrementalRebuildMode; } public int getCountIndex() { @@ -171,11 +142,11 @@ public class MaterializedViewRewritingRelVisitor extends RelVisitor { /** * Exception used to interrupt a visitor walk. */ - private static class ReturnedValue extends ControlFlowException { - private final boolean value; + private static final class ReturnedValue extends ControlFlowException { + private final IncrementalRebuildMode incrementalRebuildMode; - public ReturnedValue(boolean value) { - this.value = value; + public ReturnedValue(IncrementalRebuildMode incrementalRebuildMode) { + this.incrementalRebuildMode = incrementalRebuildMode; } } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java index 6978fd10004..941adf697ea 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java @@ -39,6 +39,7 @@ import org.apache.calcite.util.Pair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; @@ -158,7 +159,7 @@ class TestMaterializedViewsCache { return new HiveRelOptMaterialization( new DummyRel(table), new DummyRel(table), null, asList(table.getDbName(), table.getTableName()), RewriteAlgorithm.ALL, - HiveRelOptMaterialization.IncrementalRebuildMode.AVAILABLE, ParseUtils.parse(table.getViewExpandedText(), null)); + IncrementalRebuildMode.AVAILABLE, ParseUtils.parse(table.getViewExpandedText(), null)); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHiveAugmentSnapshotMaterializationRule.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHiveAugmentSnapshotMaterializationRule.java index 266f5dca92c..cbed5c06218 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHiveAugmentSnapshotMaterializationRule.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHiveAugmentSnapshotMaterializationRule.java @@ -39,7 +39,7 @@ public class TestHiveAugmentSnapshotMaterializationRule extends TestRuleBase { @Test public void testWhenSnapshotAndTableAreEmptyNoFilterAdded() { - RelNode tableScan = createT2IcebergTS(); + RelNode tableScan = createNonNativeTSSupportingSnapshots(); RelOptRule rule = HiveAugmentSnapshotMaterializationRule.with(Collections.emptyMap()); RelNode newRoot = HiveMaterializedViewUtils.applyRule(tableScan, rule); @@ -49,8 +49,8 @@ public class TestHiveAugmentSnapshotMaterializationRule extends TestRuleBase { @Test public void testWhenNoSnapshotButTableHasNewDataAFilterWithDefaultSnapshotIDAdded() { - doReturn(new SnapshotContext(42)).when(table2storageHandler).getCurrentSnapshotContext(table2); - RelNode tableScan = createT2IcebergTS(); + doReturn(new SnapshotContext(10)).when(tNNSnapshotsStorageHandler).getCurrentSnapshotContext(tNNSnapshots); + RelNode tableScan = createNonNativeTSSupportingSnapshots(); RelOptRule rule = HiveAugmentSnapshotMaterializationRule.with(Collections.emptyMap()); RelNode newRoot = HiveMaterializedViewUtils.applyRule(tableScan, rule); @@ -62,10 +62,10 @@ public class TestHiveAugmentSnapshotMaterializationRule extends TestRuleBase { @Test public void testWhenMVAndTableCurrentSnapshotAreTheSameNoFilterAdded() { - doReturn(new SnapshotContext(42)).when(table2storageHandler).getCurrentSnapshotContext(table2); - RelNode tableScan = createT2IcebergTS(); + doReturn(new SnapshotContext(42)).when(tNNSnapshotsStorageHandler).getCurrentSnapshotContext(tNNSnapshots); + RelNode tableScan = createNonNativeTSSupportingSnapshots(); Map<String, SnapshotContext> mvSnapshot = new HashMap<>(); - mvSnapshot.put(table2.getFullyQualifiedName(), new SnapshotContext(42)); + mvSnapshot.put(tNNSnapshots.getFullyQualifiedName(), new SnapshotContext(42)); RelOptRule rule = HiveAugmentSnapshotMaterializationRule.with(mvSnapshot); RelNode newRoot = HiveMaterializedViewUtils.applyRule(tableScan, rule); @@ -75,10 +75,10 @@ public class TestHiveAugmentSnapshotMaterializationRule extends TestRuleBase { @Test public void testWhenMVSnapshotIsDifferentThanTableCurrentSnapshotHasNewDataAFilterWithMVSnapshotIdAdded() { - doReturn(new SnapshotContext(10)).when(table2storageHandler).getCurrentSnapshotContext(table2); - RelNode tableScan = createT2IcebergTS(); + doReturn(new SnapshotContext(10)).when(tNNSnapshotsStorageHandler).getCurrentSnapshotContext(tNNSnapshots); + RelNode tableScan = createNonNativeTSSupportingSnapshots(); Map<String, SnapshotContext> mvSnapshot = new HashMap<>(); - mvSnapshot.put(table2.getFullyQualifiedName(), new SnapshotContext(42)); + mvSnapshot.put(tNNSnapshots.getFullyQualifiedName(), new SnapshotContext(42)); RelOptRule rule = HiveAugmentSnapshotMaterializationRule.with(mvSnapshot); RelNode newRoot = HiveMaterializedViewUtils.applyRule(tableScan, rule); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHivePushdownSnapshotFilterRule.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHivePushdownSnapshotFilterRule.java index c9569061563..a0fced83e29 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHivePushdownSnapshotFilterRule.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestHivePushdownSnapshotFilterRule.java @@ -43,7 +43,7 @@ public class TestHivePushdownSnapshotFilterRule extends TestRuleBase { @Test public void testFilterIsRemovedAndVersionIntervalFromIsSetWhenFilterHasSnapshotIdPredicate() { - RelNode tableScan = createT2IcebergTS(); + RelNode tableScan = createNonNativeTSSupportingSnapshots(); RelBuilder relBuilder = HiveRelFactories.HIVE_BUILDER.create(relOptCluster, schemaMock); RelNode root = relBuilder.push(tableScan) @@ -64,7 +64,7 @@ public class TestHivePushdownSnapshotFilterRule extends TestRuleBase { @Test public void testFilterLeftIntactWhenItDoesNotHaveSnapshotIdPredicate() { - RelNode tableScan = createT2IcebergTS(); + RelNode tableScan = createNonNativeTS(); RelBuilder relBuilder = HiveRelFactories.HIVE_BUILDER.create(relOptCluster, schemaMock); RelNode root = relBuilder.push(tableScan) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestMaterializedViewIncrementalRewritingRelVisitor.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestMaterializedViewIncrementalRewritingRelVisitor.java new file mode 100644 index 00000000000..29a8f41ed49 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestMaterializedViewIncrementalRewritingRelVisitor.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Collections; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TestMaterializedViewIncrementalRewritingRelVisitor extends TestRuleBase { + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasUnsupportedOperator() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .sort(1) // Order by is not supported + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasTSOnNonNativeTableWithSnapshots() { + RelNode ts1 = createNonNativeTS(); + when(tNonNativeStorageHandler.areSnapshotsSupported()).thenReturn(true); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + + verify(tNonNativeStorageHandler, atLeastOnce()).areSnapshotsSupported(); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasTSOnNonNativeTableWithoutSnapshots() { + RelNode ts1 = createNonNativeTS(); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + + verify(tNonNativeStorageHandler, atLeastOnce()).areSnapshotsSupported(); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasTSOnNonNativeTableSupportsSnapshots() { + RelNode ts1 = createNonNativeTSSupportingSnapshots(); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasFilter() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .filter(REX_BUILDER.makeCall(SqlStdOperatorTable.IS_NOT_NULL, REX_BUILDER.makeInputRef(ts1, 0))) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + MaterializedViewIncrementalRewritingRelVisitor.Result result = visitor.go(mvQueryPlan); + + assertThat(result.getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + assertThat(result.containsAggregate(), is(false)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasInnerJoin() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + RelNode ts2 = createTS(t2NativeMock, "t2"); + + RexNode joinCondition = REX_BUILDER.makeCall(SqlStdOperatorTable.EQUALS, + REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0), + REX_BUILDER.makeInputRef(ts2.getRowType().getFieldList().get(0).getType(), 5)); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .push(ts2) + .join(JoinRelType.INNER, joinCondition) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + } + + @Test + public void testInnerJoinWithDifferentInputs() { + RelNode tSDoesNotSupport = createNonNativeTS(); + RelNode tSSupportsInsertOnly = createNonNativeTSSupportingSnapshots(); + RelNode tSSupports = createTS(t1NativeMock, "t1"); + RelNode tSSupports2 = createTS(t2NativeMock, "t2"); + + testInnerJoin(tSDoesNotSupport, tSSupportsInsertOnly, IncrementalRebuildMode.NOT_AVAILABLE); + testInnerJoin(tSSupportsInsertOnly, tSDoesNotSupport, IncrementalRebuildMode.NOT_AVAILABLE); + testInnerJoin(tSSupportsInsertOnly, tSSupports, IncrementalRebuildMode.INSERT_ONLY); + testInnerJoin(tSSupports, tSSupportsInsertOnly, IncrementalRebuildMode.INSERT_ONLY); + testInnerJoin(tSSupports, tSSupports2, IncrementalRebuildMode.AVAILABLE); + } + + private void testInnerJoin(RelNode left, RelNode right, IncrementalRebuildMode expected) { + RexNode joinCondition = REX_BUILDER.makeCall(SqlStdOperatorTable.EQUALS, + REX_BUILDER.makeInputRef(left.getRowType().getFieldList().get(0).getType(), 0), + REX_BUILDER.makeInputRef(right.getRowType().getFieldList().get(0).getType(), 5)); + + RelNode mvQueryPlan = REL_BUILDER + .push(left) + .push(right) + .join(JoinRelType.INNER, joinCondition) + .aggregate(REL_BUILDER.groupKey(0), REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat( + String.format("leftInput=%s, rightInput=%s, expected incremental rebuild mode=%s", left, right, expected), + visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(expected)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasJoinOtherThanInner() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + RelNode ts2 = createTS(t2NativeMock, "t2"); + + RexNode joinCondition = REX_BUILDER.makeCall(SqlStdOperatorTable.EQUALS, + REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0), + REX_BUILDER.makeInputRef(ts2.getRowType().getFieldList().get(0).getType(), 5)); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .push(ts2) + .join(JoinRelType.LEFT, joinCondition) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasAggregate() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall( + SqlStdOperatorTable.SUM, + REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0))) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + MaterializedViewIncrementalRewritingRelVisitor.Result result = visitor.go(mvQueryPlan); + + assertThat(result.getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + assertThat(result.containsAggregate(), is(true)); + assertThat(result.getCountStarIndex(), is(-1)); + } + + @Test + public void testIncrementalRebuildIsAvailableWhenPlanHasAggregateAndCountStar() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall( + SqlStdOperatorTable.SUM, + REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0)), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + MaterializedViewIncrementalRewritingRelVisitor.Result result = visitor.go(mvQueryPlan); + + assertThat(result.getIncrementalRebuildMode(), is(IncrementalRebuildMode.AVAILABLE)); + assertThat(result.containsAggregate(), is(true)); + assertThat(result.getCountStarIndex(), is(1)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasAggregateAvg() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall( + SqlStdOperatorTable.AVG, + REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0))) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasAggregateAvgCountSumOfTheSameColumn() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.SUM, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.AVG, rexInputRef)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasAggregateAvgCountSumButOfDifferentColumns() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RexInputRef rexInputRef2 = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 1); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.SUM, rexInputRef2), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.AVG, rexInputRef)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasNotSupportedAggregateCall() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.STDDEV, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.SUM, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT, rexInputRef)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasBothSupportedAndNotSupportedAggregateCall() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.STDDEV, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.MIN, rexInputRef)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsInsertOnlyWhenPlanHasBothSupportedAggregateCallAndOneThatSupportsInsertOnly() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate( + REL_BUILDER.groupKey(0), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.MIN, rexInputRef), + REL_BUILDER.aggregateCall(SqlStdOperatorTable.COUNT)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.INSERT_ONLY)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasCountDistinct() { + RelNode ts1 = createTS(t1NativeMock, "t1"); + + RelDataType countRetType = + TYPE_FACTORY.createTypeWithNullability(TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT), false); + + AggregateCall aggregateCall = AggregateCall.create(SqlStdOperatorTable.COUNT, true, false, false, + Collections.emptyList(), -1, RelCollations.EMPTY, countRetType, null); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate(REL_BUILDER.groupKey(0), Collections.singletonList(aggregateCall)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + + @Test + public void testIncrementalRebuildIsNotAvailableWhenPlanHasUnsupportedAggregateOnSubPlanSupportsInsertOnly() { + RelNode ts1 = createNonNativeTSSupportingSnapshots(); + + RexInputRef rexInputRef = REX_BUILDER.makeInputRef(ts1.getRowType().getFieldList().get(0).getType(), 0); + RelNode mvQueryPlan = REL_BUILDER + .push(ts1) + .aggregate(REL_BUILDER.groupKey(), REL_BUILDER.aggregateCall(SqlStdOperatorTable.STDDEV, rexInputRef)) + .build(); + + MaterializedViewIncrementalRewritingRelVisitor visitor = new MaterializedViewIncrementalRewritingRelVisitor(); + assertThat(visitor.go(mvQueryPlan).getIncrementalRebuildMode(), is(IncrementalRebuildMode.NOT_AVAILABLE)); + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestRuleBase.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestRuleBase.java index f56e57107f9..d56794382c6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestRuleBase.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/TestRuleBase.java @@ -23,14 +23,15 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; @@ -43,16 +44,18 @@ import org.mockito.Mock; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; public class TestRuleBase { + protected static RelBuilder REL_BUILDER; protected static final RexBuilder REX_BUILDER = new RexBuilder(new JavaTypeFactoryImpl(new HiveTypeSystemImpl())); protected static final RelDataTypeFactory TYPE_FACTORY = REX_BUILDER.getTypeFactory(); @@ -78,11 +81,19 @@ public class TestRuleBase { protected static HiveStorageHandler t3NativeStorageHandler; @Mock - protected RelOptHiveTable table2Mock; - protected static RelDataType table2Type; - protected static Table table2; + protected RelOptHiveTable tNonNativeTableMock; + protected static RelDataType tNonNativeType; + protected static Table tNonNative; + + @Mock + protected RelOptHiveTable tNNSnapshotsTableMock; + protected static RelDataType tNNSnapshotsType; + protected static Table tNNSnapshots; + @Mock - protected static HiveStorageHandler table2storageHandler; + protected static HiveStorageHandler tNonNativeStorageHandler; + @Mock + protected static HiveStorageHandler tNNSnapshotsStorageHandler; @BeforeClass public static void beforeClass() throws Exception { @@ -108,12 +119,21 @@ public class TestRuleBase { put("i", SqlTypeName.INTEGER); }}, asList(VirtualColumn.ROWID, VirtualColumn.ROWISDELETED)); - table2 = createTable("t2_iceberg"); - table2Type = createTableType(new HashMap<String, SqlTypeName>() {{ + tNonNative = createTable("t_non_native"); + tNonNativeType = createTableType(new HashMap<String, SqlTypeName>() {{ + put("d", SqlTypeName.INTEGER); + put("e", SqlTypeName.VARCHAR); + put("f", SqlTypeName.INTEGER); + }}, Collections.emptyList()); + + tNNSnapshots = createTable("t_supports_snapshots"); + tNNSnapshotsType = createTableType(new HashMap<String, SqlTypeName>() {{ put("d", SqlTypeName.INTEGER); put("e", SqlTypeName.VARCHAR); put("f", SqlTypeName.INTEGER); }}, singletonList(VirtualColumn.SNAPSHOT_ID)); + + REL_BUILDER = HiveRelFactories.HIVE_BUILDER.create(relOptCluster, null); } private static Table createTable(String name) { @@ -150,13 +170,25 @@ public class TestRuleBase { lenient().doReturn(t3NativeType).when(t3NativeMock).getRowType(); lenient().doReturn(t3Native).when(t3NativeMock).getHiveTableMD(); - lenient().doReturn(table2Type).when(table2Mock).getRowType(); - lenient().doReturn(table2).when(table2Mock).getHiveTableMD(); - table2.setStorageHandler(table2storageHandler); + lenient().doReturn(tNonNativeType).when(tNonNativeTableMock).getRowType(); + lenient().doReturn(tNonNative).when(tNonNativeTableMock).getHiveTableMD(); + tNonNative.setStorageHandler(tNonNativeStorageHandler); + + lenient().doReturn(tNNSnapshotsType).when(tNNSnapshotsTableMock).getRowType(); + lenient().doReturn(tNNSnapshots).when(tNNSnapshotsTableMock).getHiveTableMD(); + tNNSnapshots.setStorageHandler(tNNSnapshotsStorageHandler); + } + + protected RelNode createNonNativeTS() { + HiveTableScan ts = createTS(tNonNativeTableMock, "t_non_native"); + lenient().doReturn(false).when(tNNSnapshotsStorageHandler).areSnapshotsSupported(); + return ts; } - protected RelNode createT2IcebergTS() { - return createTS(table2Mock, "t2"); + protected RelNode createNonNativeTSSupportingSnapshots() { + HiveTableScan ts = createTS(tNNSnapshotsTableMock, "t_supports_snapshots"); + lenient().doReturn(true).when(tNNSnapshotsStorageHandler).areSnapshotsSupported(); + return ts; } protected HiveTableScan createTS(RelOptHiveTable table, String alias) { diff --git a/ql/src/test/queries/clientpositive/show_materialized_views.q b/ql/src/test/queries/clientpositive/show_materialized_views.q index dab5d6719a4..37d9689abf9 100644 --- a/ql/src/test/queries/clientpositive/show_materialized_views.q +++ b/ql/src/test/queries/clientpositive/show_materialized_views.q @@ -23,6 +23,8 @@ CREATE MATERIALIZED VIEW shtb_aggr_view1 AS SELECT a.value, sum(a.key) FROM shtb_test1 a join shtb_test1 b on (a.key = b.key) group by a.value; CREATE MATERIALIZED VIEW shtb_aggr_view2 AS SELECT a.value, count(1), sum(a.key) FROM shtb_test1 a join shtb_test1 b on (a.key = b.key) group by a.value; +CREATE MATERIALIZED VIEW aggr_view_min AS +SELECT a.value, count(*), min(a.key) FROM shtb_test1 a join shtb_test1 b on (a.key = b.key) group by a.value; USE test2; CREATE TABLE shtb_test1(KEY INT, VALUE STRING) PARTITIONED BY(ds STRING) @@ -76,6 +78,7 @@ DROP TABLE foo_n0; USE test1; DROP MATERIALIZED VIEW shtb_test1_view1; +DROP MATERIALIZED VIEW aggr_view_min; DROP MATERIALIZED VIEW shtb_test1_view2; DROP MATERIALIZED VIEW shtb_full_view2; DROP MATERIALIZED VIEW shtb_aggr_view1; diff --git a/ql/src/test/results/clientpositive/llap/show_json_format.q.out b/ql/src/test/results/clientpositive/llap/show_json_format.q.out index 91543d3fb3e..a443ae444f5 100644 --- a/ql/src/test/results/clientpositive/llap/show_json_format.q.out +++ b/ql/src/test/results/clientpositive/llap/show_json_format.q.out @@ -75,7 +75,7 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS POSTHOOK: type: SHOWMATERIALIZEDVIEWS -{"materialized views":[{"MV Name":"mv","Rewriting Enabled":"No","Mode":"Manual refresh","Incremental rebuild":"Unknown"}]} +{"materialized views":[{"MV Name":"mv","Rewriting Enabled":"No","Mode":"Manual refresh","Incremental rebuild":"Not available"}]} PREHOOK: query: CREATE RESOURCE PLAN rp PREHOOK: type: CREATE RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest diff --git a/ql/src/test/results/clientpositive/llap/show_materialized_views.q.out b/ql/src/test/results/clientpositive/llap/show_materialized_views.q.out index 4cbf6d114af..67f08cef378 100644 --- a/ql/src/test/results/clientpositive/llap/show_materialized_views.q.out +++ b/ql/src/test/results/clientpositive/llap/show_materialized_views.q.out @@ -104,6 +104,21 @@ POSTHOOK: Output: test1@shtb_aggr_view2 POSTHOOK: Lineage: shtb_aggr_view2._c1 EXPRESSION [(shtb_test1)a.null, (shtb_test1)b.null, ] POSTHOOK: Lineage: shtb_aggr_view2._c2 EXPRESSION [(shtb_test1)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: shtb_aggr_view2.value SIMPLE [(shtb_test1)a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: CREATE MATERIALIZED VIEW aggr_view_min AS +SELECT a.value, count(*), min(a.key) FROM shtb_test1 a join shtb_test1 b on (a.key = b.key) group by a.value +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: test1@shtb_test1 +PREHOOK: Output: database:test1 +PREHOOK: Output: test1@aggr_view_min +POSTHOOK: query: CREATE MATERIALIZED VIEW aggr_view_min AS +SELECT a.value, count(*), min(a.key) FROM shtb_test1 a join shtb_test1 b on (a.key = b.key) group by a.value +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: test1@shtb_test1 +POSTHOOK: Output: database:test1 +POSTHOOK: Output: test1@aggr_view_min +POSTHOOK: Lineage: aggr_view_min._c1 EXPRESSION [(shtb_test1)a.null, (shtb_test1)b.null, ] +POSTHOOK: Lineage: aggr_view_min._c2 EXPRESSION [(shtb_test1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: aggr_view_min.value SIMPLE [(shtb_test1)a.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: USE test2 PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:test2 @@ -190,11 +205,12 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only +aggr_view_min Yes Manual refresh Available for insert operations only +shtb_aggr_view1 Yes Manual refresh Available for insert operations only shtb_aggr_view2 Yes Manual refresh Available -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS '%test%' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS '%test%' @@ -220,16 +236,16 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS '%test%' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS '%view2' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS '%view2' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild shtb_aggr_view2 Yes Manual refresh Available -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: USE test2 PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:test2 @@ -241,8 +257,8 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS 'shtb_%' POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test2_view2 No Manual refresh Unknown +shtb_test1_view1 No Manual refresh Not available +shtb_test2_view2 No Manual refresh Not available PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:default @@ -273,18 +289,19 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only +aggr_view_min Yes Manual refresh Available for insert operations only +shtb_aggr_view1 Yes Manual refresh Available for insert operations only shtb_aggr_view2 Yes Manual refresh Available -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test2_view2 No Manual refresh Unknown +shtb_test1_view1 No Manual refresh Not available +shtb_test2_view2 No Manual refresh Not available PREHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS IN test1 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: EXPLAIN SHOW MATERIALIZED VIEWS IN test1 @@ -309,11 +326,12 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_aggr_view1 Yes Manual refresh Available in presence of insert operations only +aggr_view_min Yes Manual refresh Available for insert operations only +shtb_aggr_view1 Yes Manual refresh Available for insert operations only shtb_aggr_view2 Yes Manual refresh Available -shtb_full_view2 Yes Manual refresh (Valid for 5min) Available -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_full_view2 Yes Manual refresh (Valid for 5min) Available for insert operations only +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: SHOW MATERIALIZED VIEWS IN default PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN default @@ -343,8 +361,8 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 "shtb_test%" POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -shtb_test1_view1 No Manual refresh Unknown -shtb_test1_view2 Yes Manual refresh (Valid always) Available +shtb_test1_view1 No Manual refresh Not available +shtb_test1_view2 Yes Manual refresh (Valid always) Available for insert operations only PREHOOK: query: DESCRIBE FORMATTED test1.shtb_full_view2 PREHOOK: type: DESCTABLE PREHOOK: Input: test1@shtb_full_view2 @@ -531,7 +549,7 @@ PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM `database` LIKE "fooview" POSTHOOK: type: SHOWMATERIALIZEDVIEWS # MV Name Rewriting Enabled Mode Incremental rebuild -fooview Yes Manual refresh (Valid until source tables modified) Available +fooview Yes Manual refresh (Valid until source tables modified) Available for insert operations only PREHOOK: query: DESCRIBE FORMATTED `database`.`fooview` PREHOOK: type: DESCTABLE PREHOOK: Input: database@fooview @@ -599,6 +617,14 @@ POSTHOOK: query: DROP MATERIALIZED VIEW shtb_test1_view1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: test1@shtb_test1_view1 POSTHOOK: Output: test1@shtb_test1_view1 +PREHOOK: query: DROP MATERIALIZED VIEW aggr_view_min +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: test1@aggr_view_min +PREHOOK: Output: test1@aggr_view_min +POSTHOOK: query: DROP MATERIALIZED VIEW aggr_view_min +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: test1@aggr_view_min +POSTHOOK: Output: test1@aggr_view_min PREHOOK: query: DROP MATERIALIZED VIEW shtb_test1_view2 PREHOOK: type: DROP_MATERIALIZED_VIEW PREHOOK: Input: test1@shtb_test1_view2
