This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d519c9c HIVE-21500: Disable conversion of managed table to external and vice versa at source via alter table (Sankar Hariappan, reviewed by Ashutosh Bapat, Anishek Agarwal) d519c9c is described below commit d519c9c4cba4a1f8dae5ffc2bf685464f1fe9a24 Author: Sankar Hariappan <sank...@apache.org> AuthorDate: Tue Apr 16 15:43:01 2019 +0530 HIVE-21500: Disable conversion of managed table to external and vice versa at source via alter table (Sankar Hariappan, reviewed by Ashutosh Bapat, Anishek Agarwal) Signed-off-by: Sankar Hariappan <sank...@apache.org> --- .../parse/TestReplScenariosWithStrictManaged.java | 71 ++++++++++++++++++++++ .../TestReplicationScenariosExternalTables.java | 54 ---------------- .../parse/TestReplicationWithTableMigration.java | 28 +++++++++ .../hadoop/hive/metastore/HiveAlterHandler.java | 20 ++++++ .../metastore/TransactionalValidationListener.java | 6 ++ .../metastore/utils/HiveStrictManagedUtils.java | 4 ++ 6 files changed, 129 insertions(+), 54 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java new file mode 100644 index 0000000..db042e2 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * TestReplScenariosWithStrictManaged - Test all replication scenarios with strict managed enabled + * at source and target. + */ +public class TestReplScenariosWithStrictManaged extends BaseReplicationAcrossInstances { + + @BeforeClass + public static void classLevelSetup() throws Exception { + Map<String, String> overrides = new HashMap<>(); + overrides.put(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES.varname, "true"); + overrides.put(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getHiveName(), "true"); + overrides.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); + overrides.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + + internalBeforeClassSetup(overrides, TestReplScenariosWithStrictManaged.class); + } + + @Test + public void dynamicallyConvertManagedToExternalTable() throws Throwable { + // All tables are automatically converted to ACID tables when strict managed is enabled. + // Also, it is not possible to convert ACID table to external table. + primary.run("use " + primaryDbName) + .run("create table t1 (id int) stored as orc") + .run("insert into table t1 values (1)") + .run("create table t2 (id int) partitioned by (key int) stored as orc") + .run("insert into table t2 partition(key=10) values (1)") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='true')") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='true', 'TRANSACTIONAL'='false')") + .runFailure("alter table t2 set tblproperties('EXTERNAL'='true')"); + } + + @Test + public void dynamicallyConvertExternalToManagedTable() throws Throwable { + // With Strict managed enabled, it is not possible to convert external table to ACID table. + primary.run("use " + primaryDbName) + .run("create external table t1 (id int) stored as orc") + .run("insert into table t1 values (1)") + .run("create external table t2 (place string) partitioned by (country string)") + .run("insert into table t2 partition(country='india') values ('bangalore')") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='false')") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='false', 'TRANSACTIONAL'='true')") + .runFailure("alter table t2 set tblproperties('EXTERNAL'='false')"); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java index b8e96f2..72da2f1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments; -import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -674,59 +673,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros ErrorMsg.REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID.getErrorCode()); } - @Test - public void dynamicallyConvertManagedToExternalTable() throws Throwable { - List<String> dumpWithClause = Collections.singletonList( - "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'" - ); - List<String> loadWithClause = externalTableBasePathWithClause(); - - WarehouseInstance.Tuple tupleBootstrapManagedTable = primary.run("use " + primaryDbName) - .run("create table t1 (id int)") - .run("insert into table t1 values (1)") - .run("create table t2 (id int) partitioned by (key int)") - .run("insert into table t2 partition(key=10) values (1)") - .dump(primaryDbName, null, dumpWithClause); - - replica.load(replicatedDbName, tupleBootstrapManagedTable.dumpLocation, loadWithClause); - - Hive hiveForReplica = Hive.get(replica.hiveConf); - Table replicaTable = hiveForReplica.getTable(replicatedDbName + ".t1"); - Path oldTblLocT1 = replicaTable.getDataLocation(); - - replicaTable = hiveForReplica.getTable(replicatedDbName + ".t2"); - Path oldTblLocT2 = replicaTable.getDataLocation(); - - WarehouseInstance.Tuple tupleIncConvertToExternalTbl = primary.run("use " + primaryDbName) - .run("alter table t1 set tblproperties('EXTERNAL'='true')") - .run("alter table t2 set tblproperties('EXTERNAL'='true')") - .dump(primaryDbName, tupleBootstrapManagedTable.lastReplicationId, dumpWithClause); - - assertExternalFileInfo(Arrays.asList("t1", "t2"), - new Path(tupleIncConvertToExternalTbl.dumpLocation, FILE_NAME)); - replica.load(replicatedDbName, tupleIncConvertToExternalTbl.dumpLocation, loadWithClause) - .run("use " + replicatedDbName) - .run("select id from t1") - .verifyResult("1") - .run("select id from t2 where key=10") - .verifyResult("1"); - - // Check if the table type is set correctly in target. - replicaTable = hiveForReplica.getTable(replicatedDbName + ".t1"); - assertTrue(TableType.EXTERNAL_TABLE.equals(replicaTable.getTableType())); - - replicaTable = hiveForReplica.getTable(replicatedDbName + ".t2"); - assertTrue(TableType.EXTERNAL_TABLE.equals(replicaTable.getTableType())); - - // Verify if new table location is set inside the base directory. - assertTablePartitionLocation(primaryDbName + ".t1", replicatedDbName + ".t1"); - assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2"); - - // Old location should be removed and set to new location. - assertFalse(replica.miniDFSCluster.getFileSystem().exists(oldTblLocT1)); - assertFalse(replica.miniDFSCluster.getFileSystem().exists(oldTblLocT2)); - } - private List<String> externalTableBasePathWithClause() throws IOException, SemanticException { Path externalTableLocation = new Path(REPLICA_EXTERNAL_BASE); DistributedFileSystem fileSystem = replica.miniDFSCluster.getFileSystem(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java index bafcdbe..41f2b9d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java @@ -476,4 +476,32 @@ public class TestReplicationWithTableMigration { replica.load(replicatedDbName, tuple.dumpLocation, withConfigs); verifyLoadExecution(replicatedDbName, tuple.lastReplicationId); } + + @Test + public void dynamicallyConvertManagedToExternalTable() throws Throwable { + // With Strict managed disabled but Db enabled for replication, it is not possible to convert + // external table to managed table. + primary.run("use " + primaryDbName) + .run("create table t1 (id int) clustered by(id) into 3 buckets stored as orc ") + .run("insert into t1 values(1)") + .run("create table t2 partitioned by (country string) ROW FORMAT SERDE " + + "'org.apache.hadoop.hive.serde2.avro.AvroSerDe' stored as avro " + + "tblproperties ('avro.schema.url'='" + avroSchemaFile.toUri().toString() + "')") + .run("insert into t2 partition (country='india') values ('another', 13)") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='true')") + .runFailure("alter table t2 set tblproperties('EXTERNAL'='true')"); + } + + @Test + public void dynamicallyConvertExternalToManagedTable() throws Throwable { + // With Strict managed disabled but Db enabled for replication, it is not possible to convert + // external table to managed table. + primary.run("use " + primaryDbName) + .run("create external table t1 (id int) stored as orc") + .run("insert into table t1 values (1)") + .run("create external table t2 (place string) partitioned by (country string)") + .run("insert into table t2 partition(country='india') values ('bangalore')") + .runFailure("alter table t1 set tblproperties('EXTERNAL'='false')") + .runFailure("alter table t2 set tblproperties('EXTERNAL'='false')"); + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index ad670c9..d4aaa5c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -169,6 +169,8 @@ public class HiveAlterHandler implements AlterHandler { TableName.getQualified(catName, dbname, name) + " doesn't exist"); } + checkTableTypeConversion(olddb, oldt, newt); + if (oldt.getPartitionKeysSize() != 0) { isPartitionedTable = true; } @@ -803,6 +805,24 @@ public class HiveAlterHandler implements AlterHandler { return oldParts; } + private void checkTableTypeConversion(Database db, Table oldTbl, Table newTbl) + throws InvalidOperationException { + // If the given DB is enabled for replication and strict managed is false, then table type cannot be changed. + // This is to avoid migration scenarios which causes Managed ACID table to be converted to external at replica. + // As ACID tables cannot be converted to external table and vice versa, we need to restrict this conversion at + // primary as well. + // Currently, table type conversion is allowed only between managed and external table types. + // But, to be future proof, any table type conversion is restricted on a replication enabled DB. + if (!conf.getBoolean(MetastoreConf.ConfVars.STRICT_MANAGED_TABLES.getHiveName(), false) + && !oldTbl.getTableType().equalsIgnoreCase(newTbl.getTableType()) + && ReplChangeManager.isSourceOfReplication(db)) { + throw new InvalidOperationException("Table type cannot be changed from " + oldTbl.getTableType() + + " to " + newTbl.getTableType() + " for the table " + + TableName.getQualified(oldTbl.getCatName(), oldTbl.getDbName(), oldTbl.getTableName()) + + " as it is enabled for replication."); + } + } + private boolean checkPartialPartKeysEqual(List<FieldSchema> oldPartKeys, List<FieldSchema> newPartKeys) { //return true if both are null, or false if one is null and the other isn't diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 004acf8..afa6e4c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -146,6 +146,12 @@ public final class TransactionalValidationListener extends MetaStorePreEventList } if (transactionalValuePresent) { + if (oldTable.getTableType().equals(TableType.MANAGED_TABLE.toString()) + && newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { + throw new MetaException(Warehouse.getQualifiedName(newTable) + + " cannot be converted to external table as it is transactional table."); + } + //normalize prop name parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java index 7213f8e..4610cdd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java @@ -59,6 +59,10 @@ public class HiveStrictManagedUtils { if (isAvroTableWithExternalSchema(table)) { return createValidationError(table, "Managed Avro table has externally defined schema."); } + } else if (tableType == TableType.EXTERNAL_TABLE) { + if (MetaStoreServerUtils.isTransactionalTable(table.getParameters())) { + return createValidationError(table, "Table is marked as a external table but it is transactional."); + } } }