This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 935c7d98e5c HIVE-28773: CBO fails when a materialized view is dropped
but its pre-compiled plan remains in the registry. (#6322)
935c7d98e5c is described below
commit 935c7d98e5cd8cb207cd13d917b2080f741663f2
Author: Krisztian Kasa <[email protected]>
AuthorDate: Thu Feb 19 16:47:39 2026 +0100
HIVE-28773: CBO fails when a materialized view is dropped but its
pre-compiled plan remains in the registry. (#6322)
---
.../TestHiveMaterializedViewRegistry.java | 221 +++++++++++++++++++++
.../hive/ql/txn/compactor/TestQueryRewrite.java | 94 +++++++++
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 2 +-
.../drop/DropMaterializedViewOperation.java | 2 +-
.../show/ShowMaterializedViewsFormatter.java | 2 +-
.../update/MaterializedViewUpdateOperation.java | 4 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 5 +-
.../ql/metadata/HiveMaterializedViewsRegistry.java | 113 ++++++-----
.../ql/metadata/HiveRelOptMaterialization.java | 2 +-
...zedViewsCache.java => MaterializedViewMap.java} | 107 ++++------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 25 ++-
...iewsCache.java => TestMaterializedViewMap.java} | 192 +++++++++---------
.../materialized_view_partitioned_2.q | 5 +-
.../llap/materialized_view_partitioned_2.q.out | 32 +--
14 files changed, 560 insertions(+), 246 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestHiveMaterializedViewRegistry.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestHiveMaterializedViewRegistry.java
new file mode 100644
index 00000000000..205e76c1dae
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestHiveMaterializedViewRegistry.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.hive.common.MaterializationSnapshot;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
+import org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization;
+import org.apache.hadoop.hive.ql.metadata.MaterializedViewMetadata;
+import org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import static java.util.Arrays.asList;
+import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver;
+import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriverSilently;
+
+public class TestHiveMaterializedViewRegistry extends CompactorOnTezTest {
+
+ private static final String DB = "default";
+ private static final String TABLE1 = "t1";
+ private static final String MV1 = "mat1";
+ private static final TableName MV1_NAME = new TableName("hive", DB, MV1);
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ executeStatementOnDriverSilently("drop materialized view if exists " +
MV1, driver);
+ executeStatementOnDriverSilently("drop table if exists " + TABLE1 ,
driver);
+
+ executeStatementOnDriver("create table " + TABLE1 + "(a int, b string, c
float) stored as orc TBLPROPERTIES ('transactional'='true')", driver);
+ executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values (1,
'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
+ }
+
+ @Override
+ public void tearDown() {
+ executeStatementOnDriverSilently("drop materialized view " + MV1, driver);
+ executeStatementOnDriverSilently("drop table " + TABLE1 , driver);
+
+ super.tearDown();
+ }
+
+ @Test
+ public void testRefreshAddsNewMV() throws Exception {
+ CreateMaterializedViewDesc createMaterializedViewDesc =
createMaterializedViewDesc();
+ Table mvTable = createMaterializedViewDesc.toTable(conf);
+ Hive.get().createTable(mvTable);
+
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+
+ Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+ Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+ }
+
+ @Test
+ public void testRefreshDoesNotAddMVDisabledForRewrite() throws Exception {
+ CreateMaterializedViewDesc createMaterializedViewDesc =
createMaterializedViewDesc();
+ createMaterializedViewDesc.setRewriteEnabled(false);
+ Table mvTable = createMaterializedViewDesc.toTable(conf);
+ Hive.get().createTable(mvTable);
+
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+ Assert.assertNull(materialization);
+ }
+
+ @Test
+ public void testRefreshUpdatesExistingMV() throws Exception {
+ // init the registry
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ executeStatementOnDriver("create materialized view " + MV1 + " as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
+
+ // replace the MV
+ Hive.get().dropTable(DB, MV1);
+
+ CreateMaterializedViewDesc createMaterializedViewDesc =
createMaterializedViewDesc();
+ Table mvTable = createMaterializedViewDesc.toTable(conf);
+ mvTable.setMaterializedViewMetadata(new MaterializedViewMetadata(
+ "hive", DB, MV1, new HashSet<>(), new
MaterializationSnapshot("anything")));
+ Hive.get().createTable(mvTable);
+
+ // test refreshing the registry
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+
+ Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+ Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+ Table existingMVTable =
HiveMaterializedViewUtils.extractTable(materialization);
+ Assert.assertEquals(mvTable.getTTable().getCreateTime(),
existingMVTable.getCreateTime());
+ }
+
+ @Test
+ public void testRefreshRemovesMVDisabledForRewrite() throws Exception {
+ // init the registry
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ executeStatementOnDriver("create materialized view " + MV1 + " as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
+
+ Table mvTable = Hive.get().getTable(DB, MV1);
+ mvTable.setRewriteEnabled(false);
+
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS,
StatsSetupConst.TRUE);
+ Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+ Assert.assertNull(materialization);
+ }
+
+ @Test
+ public void testRefreshAddsMVEnabledForRewrite() throws Exception {
+ // init the registry
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ executeStatementOnDriver("create materialized view " + MV1 + " disabled
rewrite as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
+
+ Table mvTable = Hive.get().getTable(DB, MV1);
+ mvTable.setRewriteEnabled(true);
+
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS,
StatsSetupConst.TRUE);
+ Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+
+ Assert.assertEquals(DB, materialization.qualifiedTableName.get(0));
+ Assert.assertEquals(MV1, materialization.qualifiedTableName.get(1));
+ }
+
+ @Test
+ public void testRefreshRemovesMVDoesNotExists() throws Exception {
+ // init the registry
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ executeStatementOnDriver("create materialized view " + MV1 + " as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
+
+ Hive.get().dropTable(DB, MV1);
+
+ HiveMaterializedViewsRegistry.get().refresh(Hive.get());
+
+ HiveRelOptMaterialization materialization =
+
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(MV1_NAME,
RewriteAlgorithm.ALL);
+ Assert.assertNull(materialization);
+ }
+
+ private static CreateMaterializedViewDesc createMaterializedViewDesc() {
+ Map<String, String> tableProperties = new HashMap<>();
+ tableProperties.put("created_with_ctas", "true");
+
+ CreateMaterializedViewDesc createMaterializedViewDesc = new
CreateMaterializedViewDesc(
+ MV1,
+ null,
+ null,
+ tableProperties,
+ null,
+ null,
+ null,
+ false,
+ true,
+ "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ null,
+ "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+ null,
+ new HashMap<>());
+
+ createMaterializedViewDesc.setViewOriginalText("select a,b,c from " +
TABLE1 + " where a > 0 or a is null");
+ createMaterializedViewDesc.setViewExpandedText("select
`t1`.`a`,`t1`.`b`,`t1`.`c` from `" + DB + "`.`" + TABLE1 + "` where `t1`.`a` >
0 or `t1`.`a` is null");
+ createMaterializedViewDesc.setCols(
+ asList(new FieldSchema("a", "int", null),
+ new FieldSchema("b", "string", null),
+ new FieldSchema("c", "float", null)));
+
+ return createMaterializedViewDesc;
+ }
+}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestQueryRewrite.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestQueryRewrite.java
new file mode 100644
index 00000000000..e49e1c26acd
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestQueryRewrite.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.execSelectAndDumpData;
+import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver;
+import static
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriverSilently;
+
+public class TestQueryRewrite extends CompactorOnTezTest {
+
+ private static final String DB = "default";
+ private static final String TABLE1 = "t1";
+ private static final String MV1 = "mat1";
+
+ private static final List<String> ORIGINAL_QUERY_PLAN = Arrays.asList(
+ "CBO PLAN:",
+ "HiveProject(a=[$0], b=[$1])",
+ " HiveFilter(condition=[>($0, 0)])",
+ " HiveTableScan(table=[[" + DB + ", " + TABLE1 + "]],
table:alias=[" + TABLE1 + "])",
+ ""
+ );
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ executeStatementOnDriverSilently("drop materialized view if exists " +
MV1, driver);
+ executeStatementOnDriverSilently("drop table if exists " + TABLE1, driver);
+
+ executeStatementOnDriver("create table " + TABLE1 + "(a int, b string, c
float) stored as orc TBLPROPERTIES ('transactional'='true')", driver);
+ executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values (1,
'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
+ executeStatementOnDriver("create materialized view " + MV1 + " stored by
iceberg tblproperties('format-version'='2') as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
+ }
+
+ @Override
+ public void tearDown() {
+ executeStatementOnDriverSilently("drop materialized view " + MV1, driver);
+ executeStatementOnDriverSilently("drop table " + TABLE1 , driver);
+
+ super.tearDown();
+ }
+
+ @Test
+ public void
testQueryIsNotRewrittenWhenMVIsDroppedButTheRegistryIsNotRefreshed() throws
Exception {
+
+ // Simulate a multi HS2 cluster.
+ // Drop the MV using a direct API call to HMS. This is similar to what
happens when the drop MV is executed by
+ // another HS2.
+ // In this case the MV is not removed from HiveMaterializedViewsRegistry
of HS2 which runs the explain query.
+ msClient.dropTable(DB, MV1);
+
+ List<String> result = execSelectAndDumpData("explain cbo select a, b from
" + TABLE1 + " where a > 0", driver, "");
+ Assert.assertEquals(ORIGINAL_QUERY_PLAN, result);
+ }
+
+ @Test
+ public void
testQueryIsNotRewrittenWhenMVIsDisabledForRewriteButTheRegistryIsNotRefreshed()
throws Exception {
+ Table mvTable = Hive.get().getTable(DB, MV1);
+ mvTable.setRewriteEnabled(false);
+
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS,
StatsSetupConst.TRUE);
+ Hive.get().alterTable(mvTable, false, environmentContext, true);
+
+ List<String> result = execSelectAndDumpData("explain cbo select a, b from
" + TABLE1 + " where a > 0", driver, "");
+ Assert.assertEquals(ORIGINAL_QUERY_PLAN, result);
+ }
+}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 64a106e08a7..61dbfdd633c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -382,7 +382,7 @@ public void clearTablesCreatedDuringTests() throws
Exception {
continue;
}
db.dropTable(dbName, tblName, true, true, fsType ==
FsType.ENCRYPTED_HDFS);
-
HiveMaterializedViewsRegistry.get().dropMaterializedView(tblObj.getDbName(),
tblObj.getTableName());
+
HiveMaterializedViewsRegistry.get().dropMaterializedView(tblObj.getFullTableName());
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/drop/DropMaterializedViewOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/drop/DropMaterializedViewOperation.java
index 9f18b2f2abc..a5ca033195c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/drop/DropMaterializedViewOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/drop/DropMaterializedViewOperation.java
@@ -54,7 +54,7 @@ public int execute() throws HiveException {
// TODO: API w/catalog name
context.getDb().dropTable(table, false);
-
HiveMaterializedViewsRegistry.get().dropMaterializedView(table.getDbName(),
table.getTableName());
+
HiveMaterializedViewsRegistry.get().dropMaterializedView(table.getFullTableName());
DDLUtils.addIfAbsentByName(new WriteEntity(table,
WriteEntity.WriteType.DDL_NO_LOCK), context);
return 0;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java
index 71f443c8dd4..2b6d8b948f0 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/show/ShowMaterializedViewsFormatter.java
@@ -137,7 +137,7 @@ void showMaterializedViews(DataOutputStream out,
List<Table> materializedViews)
@NotNull
private static String formatIncrementalRebuildMode(Table materializedView) {
HiveRelOptMaterialization relOptMaterialization =
HiveMaterializedViewsRegistry.get().
- getRewritingMaterializedView(materializedView.getDbName(),
materializedView.getTableName(), ALL);
+ getRewritingMaterializedView(materializedView.getFullTableName(), ALL);
if (relOptMaterialization == null) {
return NOT_AVAILABLE.getMessage();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
index ef91a7aff7d..a213e7ea5df 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/update/MaterializedViewUpdateOperation.java
@@ -48,8 +48,8 @@ public int execute() throws HiveException {
HiveMaterializedViewsRegistry.get().createMaterializedView(context.getDb().getConf(),
mvTable);
} else if (desc.isDisableRewrite()) {
// Disabling rewriting, removing from cache
- String[] names = desc.getName().split("\\.");
- HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0],
names[1]);
+ Table mvTable = context.getDb().getTable(desc.getName());
+
HiveMaterializedViewsRegistry.get().dropMaterializedView(mvTable.getFullTableName());
} else if (desc.isUpdateCreationMetadata()) {
// We need to update the status of the creation signature
Table mvTable = context.getDb().getTable(desc.getName());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index b3977b8c957..0a63aea1478 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -212,6 +212,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -2380,7 +2381,7 @@ private List<HiveRelOptMaterialization>
getValidMaterializedViews(
// It passed the test, load
HiveRelOptMaterialization relOptMaterialization =
HiveMaterializedViewsRegistry.get().getRewritingMaterializedView(
- materializedViewTable.getDbName(),
materializedViewTable.getTableName(), scope);
+ materializedViewTable.getFullTableName(), scope);
if (relOptMaterialization != null) {
Table cachedMaterializedViewTable =
extractTable(relOptMaterialization);
if (cachedMaterializedViewTable.equals(materializedViewTable)) {
@@ -2466,7 +2467,7 @@ public List<Table>
getAllMaterializedViewObjectsForRewriting() throws HiveExcept
public List<HiveRelOptMaterialization> getMaterializedViewsByAST(
ASTNode astNode, Set<TableName> tablesUsed, Supplier<String>
validTxnsList, HiveTxnManager txnMgr) throws HiveException {
- List<HiveRelOptMaterialization> materializedViews =
+ Collection<HiveRelOptMaterialization> materializedViews =
HiveMaterializedViewsRegistry.get().getRewritingMaterializedViews(astNode);
if (materializedViews.isEmpty()) {
return Collections.emptyList();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index f2379fb09f5..0cf02a95e39 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.metadata;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -29,6 +30,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import com.google.common.collect.ImmutableMap;
import org.apache.calcite.adapter.druid.DruidQuery;
@@ -47,6 +49,7 @@
import org.apache.calcite.rel.type.RelDataTypeImpl;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -86,6 +89,7 @@
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm.ALL;
+import static
org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils.extractTable;
/**
* Registry for materialized views. The goal of this cache is to avoid parsing
and creating
@@ -101,7 +105,7 @@ public final class HiveMaterializedViewsRegistry {
/* Singleton */
private static final HiveMaterializedViewsRegistry SINGLETON = new
HiveMaterializedViewsRegistry();
- private final MaterializedViewsCache materializedViewsCache = new
MaterializedViewsCache();
+ private final MaterializedViewMap materializedViewMap = new
MaterializedViewMap();
/* Whether the cache has been initialized or not. */
private final AtomicBoolean initialized = new AtomicBoolean(false);
@@ -168,44 +172,58 @@ private Loader(Hive db) {
@Override
public void run() {
- PerfLogger perfLogger = SessionState.getPerfLogger();
- try {
- DriverUtils.setUpAndStartSessionState(db.getConf());
- perfLogger.perfLogBegin(CLASS_NAME,
PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
- if (initialized.get()) {
- for (Table mvTable : db.getAllMaterializedViewObjectsForRewriting())
{
- RelOptMaterialization existingMV = getRewritingMaterializedView(
- mvTable.getDbName(), mvTable.getTableName(), ALL);
- if (existingMV != null) {
- // We replace if the existing MV is not newer
- Table existingMVTable =
HiveMaterializedViewUtils.extractTable(existingMV);
- if (existingMVTable.getCreateTime() < mvTable.getCreateTime() ||
- (existingMVTable.getCreateTime() == mvTable.getCreateTime()
&&
- existingMVTable.getMVMetadata().getMaterializationTime()
<= mvTable.getMVMetadata().getMaterializationTime())) {
- refreshMaterializedView(db.getConf(), existingMVTable,
mvTable);
- }
- } else {
- // Simply replace if it still does not exist
- refreshMaterializedView(db.getConf(), null, mvTable);
+ DriverUtils.setUpAndStartSessionState(db.getConf());
+ refresh(db);
+ }
+ }
+
+ public void refresh(Hive db) {
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.perfLogBegin(CLASS_NAME,
PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
+ try {
+ if (initialized.get()) {
+ Set<TableName> remaining = getRewritingMaterializedViews().stream()
+ .map(materialization ->
extractTable(materialization).getFullTableName())
+ .collect(Collectors.toSet());
+
+ for (Table mvTable : db.getAllMaterializedViewObjectsForRewriting()) {
+ RelOptMaterialization existingMV =
getRewritingMaterializedView(mvTable.getFullTableName(), ALL);
+ if (existingMV != null) {
+ // We replace if the existing MV is not newer
+ Table existingMVTable =
HiveMaterializedViewUtils.extractTable(existingMV);
+ remaining.remove(new TableName(
+ existingMVTable.getCatName(), existingMVTable.getDbName(),
existingMVTable.getTableName()));
+ if (existingMVTable.getCreateTime() < mvTable.getCreateTime() ||
+ (existingMVTable.getCreateTime() ==
mvTable.getCreateTime() &&
+
existingMVTable.getMVMetadata().getMaterializationTime() <=
mvTable.getMVMetadata().getMaterializationTime())) {
+ refreshMaterializedView(db.getConf(), existingMVTable, mvTable);
}
- }
- LOG.info("Materialized views registry has been refreshed");
- } else {
- for (Table mvTable : db.getAllMaterializedViewObjectsForRewriting())
{
+ } else {
+ // Simply replace if it still does not exist
refreshMaterializedView(db.getConf(), null, mvTable);
}
- initialized.set(true);
- LOG.info("Materialized views registry has been initialized");
}
- } catch (HiveException e) {
- if (initialized.get()) {
- LOG.error("Problem connecting to the metastore when refreshing the
view registry", e);
- } else {
- LOG.error("Problem connecting to the metastore when initializing the
view registry", e);
+
+ for (TableName tableName : remaining) {
+ dropMaterializedView(tableName);
+ }
+
+ LOG.info("Materialized views registry has been refreshed");
+ } else {
+ for (Table mvTable : db.getAllMaterializedViewObjectsForRewriting()) {
+ refreshMaterializedView(db.getConf(), null, mvTable);
}
+ initialized.set(true);
+ LOG.info("Materialized views registry has been initialized");
+ }
+ } catch (HiveException e) {
+ if (initialized.get()) {
+ LOG.error("Problem connecting to the metastore when refreshing the
view registry", e);
+ } else {
+ LOG.error("Problem connecting to the metastore when initializing the
view registry", e);
}
- perfLogger.perfLogEnd(CLASS_NAME,
PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
}
+ perfLogger.perfLogEnd(CLASS_NAME,
PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
}
public boolean isInitialized() {
@@ -265,7 +283,7 @@ public void createMaterializedView(HiveConf conf, Table
materializedViewTable) {
return;
}
- materializedViewsCache.putIfAbsent(materializedViewTable, materialization);
+ materializedViewMap.putIfAbsent(materializedViewTable, materialization);
if (LOG.isDebugEnabled()) {
LOG.debug("Created materialized view for rewriting: " +
materializedViewTable.getFullyQualifiedName());
@@ -276,8 +294,7 @@ public void createMaterializedView(HiveConf conf, Table
materializedViewTable) {
* Update the materialized view in the registry (if materialized view
exists).
*/
public void refreshMaterializedView(HiveConf conf, Table
materializedViewTable) {
- RelOptMaterialization cached = materializedViewsCache.get(
- materializedViewTable.getDbName(),
materializedViewTable.getTableName());
+ RelOptMaterialization cached =
materializedViewMap.get(materializedViewTable.getFullTableName());
if (cached == null) {
return;
}
@@ -308,7 +325,7 @@ public void refreshMaterializedView(HiveConf conf, Table
oldMaterializedViewTabl
if (newMaterialization == null) {
return;
}
- materializedViewsCache.refresh(oldMaterializedViewTable,
materializedViewTable, newMaterialization);
+ materializedViewMap.refresh(oldMaterializedViewTable,
materializedViewTable, newMaterialization);
if (LOG.isDebugEnabled()) {
LOG.debug("Materialized view refreshed: " +
materializedViewTable.getFullyQualifiedName());
@@ -319,14 +336,14 @@ public void refreshMaterializedView(HiveConf conf, Table
oldMaterializedViewTabl
* Removes the materialized view from the cache (based on table object
equality), if exists.
*/
public void dropMaterializedView(Table materializedViewTable) {
- materializedViewsCache.remove(materializedViewTable);
+ materializedViewMap.remove(materializedViewTable);
}
/**
* Removes the materialized view from the cache (based on qualified name),
if exists.
*/
- public void dropMaterializedView(String dbName, String tableName) {
- materializedViewsCache.remove(dbName, tableName);
+ public void dropMaterializedView(TableName tableName) {
+ materializedViewMap.remove(tableName);
}
/**
@@ -335,7 +352,7 @@ public void dropMaterializedView(String dbName, String
tableName) {
* @return the collection of materialized views, or the empty collection if
none
*/
List<HiveRelOptMaterialization> getRewritingMaterializedViews() {
- return materializedViewsCache.values().stream()
+ return materializedViewMap.values().stream()
.filter(materialization ->
materialization.getScope().contains(RewriteAlgorithm.CALCITE))
.collect(toList());
}
@@ -345,20 +362,24 @@ List<HiveRelOptMaterialization>
getRewritingMaterializedViews() {
*
* @return the collection of materialized views, or the empty collection if
none
*/
- public HiveRelOptMaterialization getRewritingMaterializedView(String dbName,
String viewName,
-
EnumSet<RewriteAlgorithm> scope) {
- HiveRelOptMaterialization materialization =
materializedViewsCache.get(dbName, viewName);
+ public HiveRelOptMaterialization getRewritingMaterializedView(
+ TableName viewName, Set<RewriteAlgorithm> scope) {
+ HiveRelOptMaterialization materialization =
materializedViewMap.get(viewName);
if (materialization == null) {
+ LOG.debug("Materialized view {} not found in registry", viewName);
return null;
}
if (!materialization.isSupported(scope)) {
+ LOG.debug("Materialized view {} is not available for rewrite
algorithm(s) {}", viewName, scope);
return null;
}
+
+ LOG.debug("Found materialized view {} in registry", viewName);
return materialization;
}
- public List<HiveRelOptMaterialization> getRewritingMaterializedViews(ASTNode
ast) {
- return materializedViewsCache.get(ast);
+ public Collection<HiveRelOptMaterialization>
getRewritingMaterializedViews(ASTNode ast) {
+ return materializedViewMap.get(ast);
}
private Context createContext(HiveConf conf) {
@@ -369,7 +390,7 @@ private Context createContext(HiveConf conf) {
}
public boolean isEmpty() {
- return materializedViewsCache.isEmpty();
+ return materializedViewMap.isEmpty();
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java
index 491ff137d11..18fa2ac5b9b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveRelOptMaterialization.java
@@ -71,7 +71,7 @@ public Set<RewriteAlgorithm> getScope() {
* @param scope Set of algorithms
* @return true if applicable false otherwise
*/
- public boolean isSupported(EnumSet<RewriteAlgorithm> scope) {
+ public boolean isSupported(Set<RewriteAlgorithm> scope) {
return !intersection(this.scope, scope).isEmpty();
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewMap.java
similarity index 67%
rename from
ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
rename to
ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewMap.java
index 9ffe2e3d4a9..3ebaec72223 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewMap.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hive.ql.metadata;
+import org.apache.hadoop.hive.common.TableName;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -44,21 +46,19 @@
* which provides name based lookup. The map which provides query text based
lookup is updated by lambda expressions
* passed to {@link ConcurrentHashMap#compute(Object, BiFunction)}.
*/
-public class MaterializedViewsCache {
- private static final Logger LOG =
LoggerFactory.getLogger(MaterializedViewsCache.class);
+public class MaterializedViewMap {
+ private static final Logger LOG =
LoggerFactory.getLogger(MaterializedViewMap.class);
// Key is the database name. Value a map from the qualified name to the view
object.
- private final ConcurrentMap<String, ConcurrentMap<String,
HiveRelOptMaterialization>> materializedViews =
+ private final ConcurrentMap<String, HiveRelOptMaterialization>
materializedViews =
new ConcurrentHashMap<>();
// Map for looking up materialization by view query text
private final Map<ASTKey, List<HiveRelOptMaterialization>>
sqlToMaterializedView = new ConcurrentHashMap<>();
public void putIfAbsent(Table materializedViewTable,
HiveRelOptMaterialization materialization) {
- ConcurrentMap<String, HiveRelOptMaterialization> dbMap =
ensureDbMap(materializedViewTable);
-
// You store the materialized view
- dbMap.computeIfAbsent(materializedViewTable.getTableName(), (mvTableName)
-> {
+ materializedViews.computeIfAbsent(genKey(materializedViewTable),
mvTableName -> {
List<HiveRelOptMaterialization> materializationList =
sqlToMaterializedView.computeIfAbsent(
new ASTKey(materialization.getAst()), s -> new ArrayList<>());
materializationList.add(materialization);
@@ -69,24 +69,18 @@ public void putIfAbsent(Table materializedViewTable,
HiveRelOptMaterialization m
materializedViewTable.getDbName(),
materializedViewTable.getTableName());
}
- private ConcurrentMap<String, HiveRelOptMaterialization> ensureDbMap(Table
materializedViewTable) {
- // We are going to create the map for each view in the given database
- ConcurrentMap<String, HiveRelOptMaterialization> dbMap =
- new ConcurrentHashMap<String, HiveRelOptMaterialization>();
- // If we are caching the MV, we include it in the cache
- final ConcurrentMap<String, HiveRelOptMaterialization> prevDbMap =
materializedViews.putIfAbsent(
- materializedViewTable.getDbName(), dbMap);
- if (prevDbMap != null) {
- dbMap = prevDbMap;
- }
- return dbMap;
+ private String genKey(Table materializedViewTable) {
+ return genKey(materializedViewTable.getFullTableName());
+ }
+
+ private String genKey(TableName tableName) {
+ return tableName.toString();
}
- public void refresh(
+ void refresh(
Table oldMaterializedViewTable, Table materializedViewTable,
HiveRelOptMaterialization newMaterialization) {
- ConcurrentMap<String, HiveRelOptMaterialization> dbMap =
ensureDbMap(materializedViewTable);
- dbMap.compute(materializedViewTable.getTableName(), (mvTableName,
existingMaterialization) -> {
+ materializedViews.compute(genKey(materializedViewTable), (mvTableName,
existingMaterialization) -> {
List<HiveRelOptMaterialization> optMaterializationList =
sqlToMaterializedView.computeIfAbsent(
new ASTKey(newMaterialization.getAst()), s -> new ArrayList<>());
@@ -121,23 +115,16 @@ public void refresh(
}
public void remove(Table materializedViewTable) {
- ConcurrentMap<String, HiveRelOptMaterialization> dbMap =
materializedViews.get(materializedViewTable.getDbName());
- if (dbMap != null) {
- // Delete only if the create time for the input materialized view table
and the table
- // in the map match. Otherwise, keep the one in the map.
- dbMap.computeIfPresent(materializedViewTable.getTableName(),
(mvTableName, oldMaterialization) -> {
- Table oldTable =
HiveMaterializedViewUtils.extractTable(oldMaterialization);
- if (materializedViewTable.equals(oldTable)) {
- remove(oldMaterialization, oldTable);
- return null;
- }
- return oldMaterialization;
- });
-
- if (dbMap.isEmpty()) {
- materializedViews.remove(materializedViewTable.getDbName());
+ // Delete only if the input materialized view table and the table
+ // in the map match. Otherwise, keep the one in the map.
+ materializedViews.computeIfPresent(genKey(materializedViewTable),
(mvTableName, oldMaterialization) -> {
+ Table oldTable =
HiveMaterializedViewUtils.extractTable(oldMaterialization);
+ if (materializedViewTable.equals(oldTable)) {
+ remove(oldMaterialization, oldTable);
+ return null;
}
- }
+ return oldMaterialization;
+ });
LOG.debug("Materialized view {}.{} removed from registry",
materializedViewTable.getDbName(),
materializedViewTable.getTableName());
@@ -148,50 +135,36 @@ private void remove(HiveRelOptMaterialization
materialization, Table mvTable) {
return;
}
- ASTKey ASTKey = new ASTKey(materialization.getAst());
- List<HiveRelOptMaterialization> materializationList =
sqlToMaterializedView.get(ASTKey);
+ ASTKey astKey = new ASTKey(materialization.getAst());
+ List<HiveRelOptMaterialization> materializationList =
sqlToMaterializedView.get(astKey);
if (materializationList == null) {
return;
}
materializationList.remove(materialization);
if (materializationList.isEmpty()) {
- sqlToMaterializedView.remove(ASTKey);
+ sqlToMaterializedView.remove(astKey);
}
}
- public void remove(String dbName, String tableName) {
- ConcurrentMap<String, HiveRelOptMaterialization> dbMap =
materializedViews.get(dbName);
- if (dbMap != null) {
- dbMap.computeIfPresent(tableName, (mvTableName, materialization) -> {
- remove(materialization,
HiveMaterializedViewUtils.extractTable(materialization));
- return null;
- });
-
- if (dbMap.isEmpty()) {
- materializedViews.remove(dbName);
- }
+ public void remove(TableName viewName) {
+ materializedViews.computeIfPresent(genKey(viewName), (mvTableName,
materialization) -> {
+ remove(materialization,
HiveMaterializedViewUtils.extractTable(materialization));
+ return null;
+ });
- LOG.debug("Materialized view {}.{} removed from registry", dbName,
tableName);
- }
+ LOG.debug("Materialized view {} removed from registry", viewName);
}
- public List<HiveRelOptMaterialization> values() {
- List<HiveRelOptMaterialization> result = new ArrayList<>();
- materializedViews.forEach((dbName, mvs) -> result.addAll(mvs.values()));
- return unmodifiableList(result);
+ public Collection<HiveRelOptMaterialization> values() {
+ return materializedViews.values();
}
- HiveRelOptMaterialization get(String dbName, String viewName) {
- if (materializedViews.get(dbName) != null) {
- LOG.debug("Found materialized view {}.{} in registry", dbName, viewName);
- return materializedViews.get(dbName).get(viewName);
- }
- LOG.debug("Materialized view {}.{} not found in registry", dbName,
viewName);
- return null;
+ HiveRelOptMaterialization get(TableName viewName) {
+ return materializedViews.get(genKey(viewName));
}
- public List<HiveRelOptMaterialization> get(ASTNode astNode) {
+ public Collection<HiveRelOptMaterialization> get(ASTNode astNode) {
List<HiveRelOptMaterialization> relOptMaterializationList =
sqlToMaterializedView.get(new ASTKey(astNode));
if (relOptMaterializationList == null) {
if (LOG.isTraceEnabled()) {
@@ -247,14 +220,14 @@ private boolean equals(ASTNode astNode1, ASTNode
astNode2) {
@Override
public int hashCode() {
- return hashcode(root);
+ return hashcodeOf(root);
}
- private int hashcode(ASTNode node) {
+ private int hashcodeOf(ASTNode node) {
int result = Objects.hash(node.getType(), node.getText());
for (int i = 0; i < node.getChildCount(); ++i) {
- result = 31 * result + hashcode((ASTNode) node.getChild(i));
+ result = 31 * result + hashcodeOf((ASTNode) node.getChild(i));
}
return result;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index d9665b2417d..c2db9f6887f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -2085,7 +2085,7 @@ private RelNode rewriteUsingViews(RelOptPlanner planner,
RelNode basePlan,
for (RelOptMaterialization materialization : materializations) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding materialization {} to the planner; the plan
is:\n{}", materialization.qualifiedTableName,
- RelOptUtil.toString(materialization.queryRel));
+ RelOptUtil.toString(materialization.queryRel));
}
planner.addMaterialization(materialization);
}
@@ -2105,20 +2105,25 @@ private RelNode rewriteUsingViews(RelOptPlanner
planner, RelNode basePlan,
final String ruleExclusionRegex =
conf.get(ConfVars.HIVE_CBO_RULE_EXCLUSION_REGEX.varname, "");
if (!ruleExclusionRegex.isEmpty()) {
LOG.info("The CBO rules matching the following regex are excluded from
planning: {}",
- ruleExclusionRegex);
+ ruleExclusionRegex);
planner.setRuleDescExclusionFilter(Pattern.compile(ruleExclusionRegex));
}
planner.setRoot(basePlan);
- basePlan = planner.findBestExp();
- // Remove view-based rewriting rules from planner
- planner.clear();
+ try {
+ return planner.findBestExp();
+ } catch (Exception ex) {
+ LOG.warn("Error while performing materialized view based query
rewrite", ex);
+ return basePlan;
+ } finally {
+ // Remove view-based rewriting rules from planner
+ planner.clear();
- // Restore default cost model
- optCluster.invalidateMetadataQuery();
-
RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider));
+ // Restore default cost model
+ optCluster.invalidateMetadataQuery();
+
RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider));
- perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: View-based rewriting");
- return basePlan;
+ perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: View-based rewriting");
+ }
}
private RelNode applyCteRewriting(RelOptPlanner planner, RelNode
basePlan, RelMetadataProvider mdProvider, RexExecutor executorProvider) {
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewMap.java
similarity index 59%
rename from
ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
rename to
ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewMap.java
index fa0f1c59535..a056a916f05 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewMap.java
@@ -36,8 +36,6 @@
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.util.Litmus;
import org.apache.calcite.util.Pair;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.IncrementalRebuildMode;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -50,7 +48,7 @@
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
-import java.util.EnumSet;
+import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -67,11 +65,11 @@
import static org.hamcrest.core.Is.is;
/**
- * Functional and parallel execution tests for {@link MaterializedViewsCache}.
+ * Functional and parallel execution tests for {@link MaterializedViewMap}.
* Parallel execution test is disabled by default.
*/
-class TestMaterializedViewsCache {
- private MaterializedViewsCache materializedViewsCache;
+class TestMaterializedViewMap {
+ private MaterializedViewMap materializedViewMap;
private Table defaultMV1;
private HiveRelOptMaterialization defaultMaterialization1;
private Table defaultMV1Same;
@@ -92,12 +90,12 @@ void setUp() throws ParseException {
db1MV1 = getTable("db1", "mat1", "select col0 from t1 where col0 = 'foo'");
db1Materialization1 = createMaterialization(db1MV1);
- materializedViewsCache = new MaterializedViewsCache();
+ materializedViewMap = new MaterializedViewMap();
}
@Test
void testEmptyCache() {
- MaterializedViewsCache emptyCache = new MaterializedViewsCache();
+ MaterializedViewMap emptyCache = new MaterializedViewMap();
ASTNode any = (ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier,
"any");
@@ -108,43 +106,43 @@ void testEmptyCache() {
@Test
void testGetByTableNameFromEmptyCache() {
- MaterializedViewsCache emptyCache = new MaterializedViewsCache();
+ MaterializedViewMap emptyCache = new MaterializedViewMap();
- assertThat(emptyCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(nullValue()));
+ assertThat(emptyCache.get(defaultMV1.getFullTableName()), is(nullValue()));
}
@Test
void testQueryDoesNotMatchAnyMVDefinition() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
ASTNode notFound = (ASTNode)
ParseDriver.adaptor.create(HiveParser.Identifier, "notFound");
- assertThat(materializedViewsCache.get(notFound).isEmpty(), is(true));
- assertThat(materializedViewsCache.values().size(), is(1));
+ assertThat(materializedViewMap.get(notFound).isEmpty(), is(true));
+ assertThat(materializedViewMap.values().size(), is(1));
}
@Test
void testAdd() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).get(0),
is(defaultMaterialization1));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(1));
- assertThat(materializedViewsCache.values().get(0),
is(defaultMaterialization1));
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).stream().findFirst().get(),
is(defaultMaterialization1));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(1));
+ assertThat(materializedViewMap.values().stream().findFirst().get(),
is(defaultMaterialization1));
}
@Test
void testAddSameMVTwice() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).get(0),
is(defaultMaterialization1));
- assertThat(materializedViewsCache.values().size(), is(1));
- assertThat(materializedViewsCache.values().get(0),
is(defaultMaterialization1));
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).stream().findFirst().get(),
is(defaultMaterialization1));
+ assertThat(materializedViewMap.values().size(), is(1));
+ assertThat(materializedViewMap.values().stream().findFirst().get(),
is(defaultMaterialization1));
}
private Table getTable(String db, String tableName, String definition) {
@@ -164,105 +162,105 @@ private static HiveRelOptMaterialization
createMaterialization(Table table) thro
@Test
void testAddMVsWithSameDefinition() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.putIfAbsent(defaultMV1Same,
defaultMaterialization1Same);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
- assertThat(materializedViewsCache.get(defaultMV1Same.getDbName(),
defaultMV1Same.getTableName()), is(defaultMaterialization1Same));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(2));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1Same));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(2));
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1Same,
defaultMaterialization1Same);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+ assertThat(materializedViewMap.get(defaultMV1Same.getFullTableName()),
is(defaultMaterialization1Same));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(2));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1Same));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(2));
}
@Test
void testAddMVsWithSameDefinitionButDifferentDatabase() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.putIfAbsent(db1MV1, db1Materialization1);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
- assertThat(materializedViewsCache.get(db1MV1.getDbName(),
db1MV1.getTableName()), is(db1Materialization1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(2));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(db1Materialization1));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(2));
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(db1MV1, db1Materialization1);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+ assertThat(materializedViewMap.get(db1MV1.getFullTableName()),
is(db1Materialization1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(2));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(db1Materialization1));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(2));
}
@Test
void testLookupByTextIsCaseSensitive() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.putIfAbsent(defaultMVUpCase,
defaultMaterializationUpCase);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMVUpCase,
defaultMaterializationUpCase);
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(2));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(2));
}
@Test
void testRefreshWhenMVWasNotCached() {
- materializedViewsCache.refresh(defaultMV1, defaultMV1,
defaultMaterialization1);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(1));
- assertThat(materializedViewsCache.values(),
hasItem(defaultMaterialization1));
+ materializedViewMap.refresh(defaultMV1, defaultMV1,
defaultMaterialization1);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(1));
+ assertThat(materializedViewMap.values(), hasItem(defaultMaterialization1));
}
@Test
void testRefreshWhenMVIsCachedButWasUpdated() throws ParseException {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
HiveRelOptMaterialization newMaterialization =
createMaterialization(defaultMV1);
- materializedViewsCache.refresh(defaultMV1, defaultMV1, newMaterialization);
+ materializedViewMap.refresh(defaultMV1, defaultMV1, newMaterialization);
assertThat(newMaterialization, is(not(defaultMaterialization1)));
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(newMaterialization));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(newMaterialization));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(1));
- assertThat(materializedViewsCache.values(), hasItem(newMaterialization));
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(newMaterialization));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(newMaterialization));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(1));
+ assertThat(materializedViewMap.values(), hasItem(newMaterialization));
}
@Test
void testRefreshWhenMVRefersToANewMaterialization() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.refresh(defaultMV1Same, defaultMV1,
defaultMaterialization1);
-
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(defaultMaterialization1));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).size(),
is(1));
- assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
- assertThat(materializedViewsCache.isEmpty(), is(false));
- assertThat(materializedViewsCache.values().size(), is(1));
- assertThat(materializedViewsCache.values(),
hasItem(defaultMaterialization1));
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.refresh(defaultMV1Same, defaultMV1,
defaultMaterialization1);
+
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(defaultMaterialization1));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).size(),
is(1));
+ assertThat(materializedViewMap.get(defaultMaterialization1.getAst()),
hasItem(defaultMaterialization1));
+ assertThat(materializedViewMap.isEmpty(), is(false));
+ assertThat(materializedViewMap.values().size(), is(1));
+ assertThat(materializedViewMap.values(), hasItem(defaultMaterialization1));
}
@Test
void testRemoveByTable() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.remove(defaultMV1);
+ materializedViewMap.remove(defaultMV1);
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(nullValue()));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).isEmpty(),
is(true));
- assertThat(materializedViewsCache.isEmpty(), is(true));
- assertThat(materializedViewsCache.values().isEmpty(), is(true));
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(nullValue()));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).isEmpty(),
is(true));
+ assertThat(materializedViewMap.isEmpty(), is(true));
+ assertThat(materializedViewMap.values().isEmpty(), is(true));
}
@Test
void testRemoveByTableName() {
- materializedViewsCache.putIfAbsent(defaultMV1, defaultMaterialization1);
+ materializedViewMap.putIfAbsent(defaultMV1, defaultMaterialization1);
- materializedViewsCache.remove(defaultMV1.getDbName(),
defaultMV1.getTableName());
+ materializedViewMap.remove(defaultMV1.getFullTableName());
- assertThat(materializedViewsCache.get(defaultMV1.getDbName(),
defaultMV1.getTableName()), is(nullValue()));
-
assertThat(materializedViewsCache.get(defaultMaterialization1.getAst()).isEmpty(),
is(true));
- assertThat(materializedViewsCache.isEmpty(), is(true));
- assertThat(materializedViewsCache.values().isEmpty(), is(true));
+ assertThat(materializedViewMap.get(defaultMV1.getFullTableName()),
is(nullValue()));
+
assertThat(materializedViewMap.get(defaultMaterialization1.getAst()).isEmpty(),
is(true));
+ assertThat(materializedViewMap.isEmpty(), is(true));
+ assertThat(materializedViewMap.values().isEmpty(), is(true));
}
@Disabled("Testing parallelism only")
@@ -291,15 +289,15 @@ void testParallelism() throws ParseException {
List<Callable<Void>> callableList = new ArrayList<>();
callableList.add(() -> {
for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
- materializedViewsCache.refresh(entry.left, entry.left, entry.right);
+ materializedViewMap.refresh(entry.left, entry.left, entry.right);
}
return null;
});
callableList.add(() -> {
for (int j = 0; j < ITERATIONS; ++j) {
for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
- materializedViewsCache.remove(entry.left);
- materializedViewsCache.putIfAbsent(entry.left, entry.right);
+ materializedViewMap.remove(entry.left);
+ materializedViewMap.putIfAbsent(entry.left, entry.right);
}
}
return null;
@@ -307,14 +305,14 @@ void testParallelism() throws ParseException {
for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
callableList.add(() -> {
for (int j = 0; j < ITERATIONS; ++j) {
- materializedViewsCache.get(entry.right.getAst());
+ materializedViewMap.get(entry.right.getAst());
}
return null;
});
}
callableList.add(() -> {
for (int j = 0; j < ITERATIONS; ++j) {
- List<HiveRelOptMaterialization> materializations =
materializedViewsCache.values();
+ Collection<HiveRelOptMaterialization> materializations =
materializedViewMap.values();
}
return null;
});
diff --git
a/ql/src/test/queries/clientpositive/materialized_view_partitioned_2.q
b/ql/src/test/queries/clientpositive/materialized_view_partitioned_2.q
index f617eaff577..a56f397c1f9 100644
--- a/ql/src/test/queries/clientpositive/materialized_view_partitioned_2.q
+++ b/ql/src/test/queries/clientpositive/materialized_view_partitioned_2.q
@@ -32,6 +32,9 @@ SELECT * FROM src_txn_2 where value > 'val_220' and value <
'val_230';
SELECT * FROM src_txn_2 where value > 'val_220' and value < 'val_230';
+DROP MATERIALIZED VIEW partition_mv_1;
+DROP MATERIALIZED VIEW partition_mv_4;
+
-- SHOULD CHOOSE partition_mv_1, partition_mv_3 OR partition_mv_4
-- SINCE IT IS THE MOST EFFICIENT READING ONLY ONE PARTITION
EXPLAIN
@@ -39,8 +42,6 @@ SELECT * FROM src_txn_2 where key > 224 and key < 226;
SELECT * FROM src_txn_2 where key > 223 and key < 225;
-DROP MATERIALIZED VIEW partition_mv_1;
DROP MATERIALIZED VIEW partition_mv_2;
DROP MATERIALIZED VIEW partition_mv_3;
-DROP MATERIALIZED VIEW partition_mv_4;
DROP TABLE src_txn_2;
diff --git
a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out
b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out
index 38baac075e9..21bceb01fde 100644
---
a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out
+++
b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_2.q.out
@@ -530,6 +530,22 @@ POSTHOOK: Input: default@src_txn_2
228 val_228
229 val_229
229 val_229
+PREHOOK: query: DROP MATERIALIZED VIEW partition_mv_1
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@partition_mv_1
+PREHOOK: Output: default@partition_mv_1
+POSTHOOK: query: DROP MATERIALIZED VIEW partition_mv_1
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@partition_mv_1
+POSTHOOK: Output: default@partition_mv_1
+PREHOOK: query: DROP MATERIALIZED VIEW partition_mv_4
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@partition_mv_4
+PREHOOK: Output: default@partition_mv_4
+POSTHOOK: query: DROP MATERIALIZED VIEW partition_mv_4
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@partition_mv_4
+POSTHOOK: Output: default@partition_mv_4
PREHOOK: query: EXPLAIN
SELECT * FROM src_txn_2 where key > 224 and key < 226
PREHOOK: type: QUERY
@@ -574,14 +590,6 @@ POSTHOOK: Input: default@src_txn_2
#### A masked pattern was here ####
224 val_224
224 val_224
-PREHOOK: query: DROP MATERIALIZED VIEW partition_mv_1
-PREHOOK: type: DROP_MATERIALIZED_VIEW
-PREHOOK: Input: default@partition_mv_1
-PREHOOK: Output: default@partition_mv_1
-POSTHOOK: query: DROP MATERIALIZED VIEW partition_mv_1
-POSTHOOK: type: DROP_MATERIALIZED_VIEW
-POSTHOOK: Input: default@partition_mv_1
-POSTHOOK: Output: default@partition_mv_1
PREHOOK: query: DROP MATERIALIZED VIEW partition_mv_2
PREHOOK: type: DROP_MATERIALIZED_VIEW
PREHOOK: Input: default@partition_mv_2
@@ -598,14 +606,6 @@ POSTHOOK: query: DROP MATERIALIZED VIEW partition_mv_3
POSTHOOK: type: DROP_MATERIALIZED_VIEW
POSTHOOK: Input: default@partition_mv_3
POSTHOOK: Output: default@partition_mv_3
-PREHOOK: query: DROP MATERIALIZED VIEW partition_mv_4
-PREHOOK: type: DROP_MATERIALIZED_VIEW
-PREHOOK: Input: default@partition_mv_4
-PREHOOK: Output: default@partition_mv_4
-POSTHOOK: query: DROP MATERIALIZED VIEW partition_mv_4
-POSTHOOK: type: DROP_MATERIALIZED_VIEW
-POSTHOOK: Input: default@partition_mv_4
-POSTHOOK: Output: default@partition_mv_4
PREHOOK: query: DROP TABLE src_txn_2
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@src_txn_2
