This is an automated email from the ASF dual-hosted git repository. anishek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 18893d7 HIVE-24067:TestReplicationScenariosExclusiveReplica - Wrong FS error during DB drop (Pravin Kumar Sinha, reviewed by Aasha Medhi) 18893d7 is described below commit 18893d7e0bbde4e596c00e48560ddc15da532076 Author: Anishek Agarwal <anis...@gmail.com> AuthorDate: Wed Aug 26 14:00:48 2020 +0530 HIVE-24067:TestReplicationScenariosExclusiveReplica - Wrong FS error during DB drop (Pravin Kumar Sinha, reviewed by Aasha Medhi) --- .../ql/parse/BaseReplicationAcrossInstances.java | 34 +++++++------- .../TestReplicationScenariosExclusiveReplica.java | 54 ++++++++++++++++++++++ 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java index 5fb44dd..476a940 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java @@ -63,7 +63,8 @@ public class BaseReplicationAcrossInstances { put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); }}; localOverrides.putAll(overrides); - setReplicaExternalBase(miniDFSCluster.getFileSystem(), localOverrides); + setFullyQualifiedReplicaExternalTableBase(miniDFSCluster.getFileSystem()); + localOverrides.put(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname, fullyQualifiedReplicaExternalBase); primary = new WarehouseInstance(LOG, miniDFSCluster, localOverrides); localOverrides.put(MetastoreConf.ConfVars.REPLDIR.getHiveName(), primary.repldDir); replica = new WarehouseInstance(LOG, miniDFSCluster, localOverrides); @@ -73,9 +74,7 @@ public class BaseReplicationAcrossInstances { static void internalBeforeClassSetupExclusiveReplica(Map<String, String> primaryOverrides, Map<String, String> replicaOverrides, Class clazz) throws Exception { - /** - * Setup replica cluster. - */ + // Setup replica HDFS. String replicaBaseDir = Files.createTempDirectory("replica").toFile().getAbsolutePath(); replicaConf = new HiveConf(clazz); replicaConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, replicaBaseDir); @@ -84,28 +83,30 @@ public class BaseReplicationAcrossInstances { MiniDFSCluster miniReplicaDFSCluster = new MiniDFSCluster.Builder(replicaConf).numDataNodes(1).format(true).build(); - Map<String, String> localOverrides = new HashMap<>(); - localOverrides.put("fs.defaultFS", miniReplicaDFSCluster.getFileSystem().getUri().toString()); - localOverrides.put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); - localOverrides.putAll(replicaOverrides); - setReplicaExternalBase(miniReplicaDFSCluster.getFileSystem(), localOverrides); - replica = new WarehouseInstance(LOG, miniReplicaDFSCluster, localOverrides); - - /** - * Setup primary cluster. - */ + // Setup primary HDFS. String primaryBaseDir = Files.createTempDirectory("base").toFile().getAbsolutePath(); conf = new HiveConf(clazz); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, primaryBaseDir); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniPrimaryDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); - localOverrides.clear(); + + // Setup primary warehouse. + setFullyQualifiedReplicaExternalTableBase(miniReplicaDFSCluster.getFileSystem()); + Map<String, String> localOverrides = new HashMap<>(); localOverrides.put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); localOverrides.put(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname, fullyQualifiedReplicaExternalBase); localOverrides.put("fs.defaultFS", miniPrimaryDFSCluster.getFileSystem().getUri().toString()); localOverrides.putAll(primaryOverrides); primary = new WarehouseInstance(LOG, miniPrimaryDFSCluster, localOverrides); + + // Setup replica warehouse. + localOverrides.clear(); + localOverrides.put(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname, fullyQualifiedReplicaExternalBase); + localOverrides.put("fs.defaultFS", miniReplicaDFSCluster.getFileSystem().getUri().toString()); + localOverrides.put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); + localOverrides.putAll(replicaOverrides); + replica = new WarehouseInstance(LOG, miniReplicaDFSCluster, localOverrides); } @AfterClass @@ -114,10 +115,9 @@ public class BaseReplicationAcrossInstances { replica.close(); } - private static void setReplicaExternalBase(FileSystem fs, Map<String, String> confMap) throws IOException { + private static void setFullyQualifiedReplicaExternalTableBase(FileSystem fs) throws IOException { fs.mkdirs(REPLICA_EXTERNAL_BASE); fullyQualifiedReplicaExternalBase = fs.getFileStatus(REPLICA_EXTERNAL_BASE).getPath().toString(); - confMap.put(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname, fullyQualifiedReplicaExternalBase); } @Before diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java index 7e6b5e2..549447e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExclusiveReplica.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.parse; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -24,6 +25,7 @@ import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncod import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -175,6 +177,58 @@ public class TestReplicationScenariosExclusiveReplica extends BaseReplicationAcr .verifyResult("800"); } + @Test + public void externalTableReplicationDropDatabase() throws Throwable { + String primaryDb = "primarydb1"; + String replicaDb = "repldb1"; + String tableName = "t1"; + List<String> withClauseOptions = getStagingLocationConfig(primary.repldDir); + WarehouseInstance.Tuple tuple = primary + .run("create database " + primaryDb) + .run("alter database "+ primaryDb + " set dbproperties('repl.source.for'='1,2,3')") + .run("use " + primaryDb) + .run("create external table " + tableName + " (id int)") + .run("insert into table " + tableName + " values (500)") + .dump(primaryDb, withClauseOptions); + + replica.load(replicaDb, primaryDb, withClauseOptions) + .run("use " + replicaDb) + .run("show tables like '" + tableName + "'") + .verifyResult(tableName) + .run("select id from " + tableName) + .verifyResult("500"); + + Path dbDataLocPrimary = new Path(primary.externalTableWarehouseRoot, primaryDb + ".db"); + Path extTableBase = new Path(replica.getConf().get(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname)); + Path dbDataLocReplica = new Path(extTableBase + dbDataLocPrimary.toUri().getPath()); + verifyTableDataExists(primary, dbDataLocPrimary, tableName, true); + verifyTableDataExists(replica, dbDataLocReplica, tableName, true); + + primary.run("show databases like '" + primaryDb + "'") + .verifyResult(primaryDb); + replica.run("show databases like '" + replicaDb + "'") + .verifyResult(replicaDb); + primary.run("drop database " + primaryDb + " cascade"); + replica.run("drop database " + replicaDb + " cascade"); + primary.run("show databases like '" + primaryDb + "'") + .verifyResult(null); + replica.run("show databases like '" + replicaDb + "'") + .verifyResult(null); + + verifyTableDataExists(primary, dbDataLocPrimary, tableName, false); + verifyTableDataExists(replica, dbDataLocReplica, tableName, true); + } + + private void verifyTableDataExists(WarehouseInstance warehouse, Path dbDataPath, String tableName, + boolean shouldExists) throws IOException { + FileSystem fileSystem = FileSystem.get(warehouse.warehouseRoot.toUri(), warehouse.getConf()); + Path tablePath = new Path(dbDataPath, tableName); + Path dataFilePath = new Path(tablePath, "000000_0"); + Assert.assertEquals(shouldExists, fileSystem.exists(dbDataPath)); + Assert.assertEquals(shouldExists, fileSystem.exists(tablePath)); + Assert.assertEquals(shouldExists, fileSystem.exists(dataFilePath)); + } + private List<String> getStagingLocationConfig(String stagingLoc) { List<String> confList = new ArrayList<>(); confList.add("'" + HiveConf.ConfVars.REPLDIR.varname + "'='" + stagingLoc + "'");