This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push: new e61cd0f12f1 HIVE-27379: Backport HIVE-22566 : Drop table involved in materialized view leaves the table in inconsistent state (Pablo Junge, reviewed by Miklos Gergely) e61cd0f12f1 is described below commit e61cd0f12f1be76ebf2b3383d0fbe66af8ab3296 Author: Diksha628 <43694846+diksha...@users.noreply.github.com> AuthorDate: Wed Jun 7 15:58:12 2023 +0530 HIVE-27379: Backport HIVE-22566 : Drop table involved in materialized view leaves the table in inconsistent state (Pablo Junge, reviewed by Miklos Gergely) Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#4359) --- .../hcatalog/listener/DummyRawStoreFailEvent.java | 5 ++ .../org/apache/hadoop/hive/ql/metadata/Hive.java | 7 --- .../clientnegative/drop_table_used_by_mv2.q | 12 ++++ .../clientnegative/drop_table_used_by_mv.q.out | 3 +- .../clientnegative/drop_table_used_by_mv2.q.out | 72 ++++++++++++++++++++++ .../hadoop/hive/metastore/HiveMetaStore.java | 9 +++ .../apache/hadoop/hive/metastore/ObjectStore.java | 35 +++++++++++ .../org/apache/hadoop/hive/metastore/RawStore.java | 10 +++ .../hadoop/hive/metastore/cache/CachedStore.java | 5 ++ .../metastore/DummyRawStoreControlledCommit.java | 5 ++ .../metastore/DummyRawStoreForJdoConnection.java | 5 ++ 11 files changed, 160 insertions(+), 8 deletions(-) diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 33f2ea09e57..480cdc3125b 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -261,6 +261,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } } + @Override + public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) { + return objectStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @Override public Table getTable(String catName, String dbName, String tableName) throws MetaException { return objectStore.getTable(catName, dbName, tableName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index fc969fc651c..faeeb864a69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1039,13 +1039,6 @@ public class Hive { if (!ignoreUnknownTab) { throw new HiveException(e); } - } catch (MetaException e) { - int idx = ExceptionUtils.indexOfType(e, SQLIntegrityConstraintViolationException.class); - if (idx != -1 && ExceptionUtils.getThrowables(e)[idx].getMessage().contains("MV_TABLES_USED")) { - throw new HiveException("Cannot drop table since it is used by at least one materialized view definition. " + - "Please drop any materialized view that uses the table before dropping it", e); - } - throw new HiveException(e); } catch (Exception e) { throw new HiveException(e); } diff --git a/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q b/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q new file mode 100644 index 00000000000..458cc9ea942 --- /dev/null +++ b/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q @@ -0,0 +1,12 @@ +create table mytable (key int, value string); +insert into mytable values (1, 'val1'), (2, 'val2'); +create view myview as select * from mytable; + +create materialized view mv1 disable rewrite as +select key, value from myview; +create materialized view mv2 disable rewrite as +select count(*) from myview; + +-- dropping the view is fine, as the MV uses not the view itself, but it's query for creating it's own during it's creation +drop view myview; +drop table mytable; diff --git a/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out b/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out index 88e3b7dcdef..d35e9fa976d 100644 --- a/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out +++ b/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out @@ -32,4 +32,5 @@ PREHOOK: query: drop table mytable PREHOOK: type: DROPTABLE PREHOOK: Input: default@mytable PREHOOK: Output: default@mytable -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot drop table since it is used by at least one materialized view definition. Please drop any materialized view that uses the table before dropping it +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1] +) diff --git a/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out b/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out new file mode 100644 index 00000000000..fd6faa5d8dd --- /dev/null +++ b/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: create table mytable (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mytable +POSTHOOK: query: create table mytable (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mytable +PREHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@mytable +POSTHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@mytable +POSTHOOK: Lineage: mytable.key SCRIPT [] +POSTHOOK: Lineage: mytable.value SCRIPT [] +PREHOOK: query: create view myview as select * from mytable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@mytable +PREHOOK: Output: database:default +PREHOOK: Output: default@myview +POSTHOOK: query: create view myview as select * from mytable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myview +POSTHOOK: Lineage: myview.key SIMPLE [(mytable)mytable.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: myview.value SIMPLE [(mytable)mytable.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv2 +POSTHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv2 +PREHOOK: query: drop view myview +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@myview +PREHOOK: Output: default@myview +POSTHOOK: query: drop view myview +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@myview +POSTHOOK: Output: default@myview +PREHOOK: query: drop table mytable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@mytable +PREHOOK: Output: default@mytable +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1, default.mv2] +) diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 8270d8bf282..629666ac000 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2536,6 +2536,15 @@ public class HiveMetaStore extends ThriftHiveMetastore { if (tbl == null) { throw new NoSuchObjectException(name + " doesn't exist"); } + + // Check if table is part of a materialized view. + // If it is, it cannot be dropped. + List<String> isPartOfMV = ms.isPartOfMaterializedView(catName, dbname, name); + if (!isPartOfMV.isEmpty()) { + throw new MetaException(String.format("Cannot drop table as it is used in the following materialized" + + " views %s%n", isPartOfMV)); + } + if (tbl.getSd() == null) { throw new MetaException("Table metadata is corrupted"); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 4f02e7b8325..9b11e4836c3 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1238,6 +1238,41 @@ public class ObjectStore implements RawStore, Configurable { return success; } + @Override + public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) { + + boolean committed = false; + Query query = null; + List<String> mViewList = new ArrayList<>(); + + try { + openTransaction(); + + query = pm.newQuery("select from org.apache.hadoop.hive.metastore.model.MCreationMetadata"); + + List<MCreationMetadata> creationMetadata = (List<MCreationMetadata>)query.execute(); + Iterator<MCreationMetadata> iter = creationMetadata.iterator(); + + while (iter.hasNext()) + { + MCreationMetadata p = iter.next(); + Set<MTable> tables = p.getTables(); + for (MTable table : tables) { + if (dbName.equals(table.getDatabase().getName()) && tblName.equals(table.getTableName())) { + LOG.info("Cannot drop table " + table.getTableName() + + " as it is being used by MView " + p.getTblName()); + mViewList.add(p.getDbName() + "." + p.getTblName()); + } + } + } + + committed = commitTransaction(); + } finally { + rollbackAndCleanup(committed, query); + } + return mViewList; + } + private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName( String catName, String dbName, String tableName, String constraintname) { catName = normalizeIdentifier(catName); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 2bd4eb5b688..dba07cd56f9 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1672,4 +1672,14 @@ public interface RawStore extends Configurable { * @param tableName the name of the table for which the dump is being taken */ List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException; + + /** + * Checking if table is part of a materialized view. + * @param catName catalog the table is in + * @param dbName database the table is in + * @param tblName table name + * @return list of materialized views that uses the table + */ + List<String> isPartOfMaterializedView(String catName, String dbName, String tblName); + } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 39cba08eebf..cde755537a2 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2491,6 +2491,11 @@ public class CachedStore implements RawStore, Configurable { && MetastoreConf.getAsString(conf, MetastoreConf.ConfVars.CACHED_RAW_STORE_CACHED_OBJECTS_BLACKLIST).isEmpty(); } + @Override + public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) { + return rawStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @VisibleForTesting void resetCatalogCache() { sharedCache.resetCatalogCache(); diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index e57fcf2d9e3..974a9d61236 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1193,6 +1193,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { objectStore.cleanWriteNotificationEvents(olderThan); } + @Override + public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) { + return objectStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @Override public List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException { return objectStore.getAllWriteEventInfo(txnId, dbName, tableName); diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 46548e5a920..9af8142bda7 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1198,6 +1198,11 @@ public class DummyRawStoreForJdoConnection implements RawStore { public void cleanWriteNotificationEvents(int olderThan) { } + @Override + public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) { + throw new RuntimeException("unimplemented"); + } + @Override public List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException { return null;