Repository: hive Updated Branches: refs/heads/master 0a5b3b4d3 -> 368d9cffe
HIVE-19881: Allow metadata-only dump for database which are not source of replication (Mahesh Kumar Behera, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/368d9cff Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/368d9cff Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/368d9cff Branch: refs/heads/master Commit: 368d9cffe77bf6b7076ca83ec300c69fde651cb7 Parents: 0a5b3b4 Author: Sankar Hariappan <sank...@apache.org> Authored: Fri Jun 15 08:46:22 2018 +0530 Committer: Sankar Hariappan <sank...@apache.org> Committed: Fri Jun 15 08:46:22 2018 +0530 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 4 +++ ...TestReplicationScenariosAcrossInstances.java | 36 ++++++++++++++++++++ .../ql/parse/ReplicationSemanticAnalyzer.java | 29 +++++++++------- 3 files changed, 57 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/368d9cff/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index f4cdf02..862140f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -3162,6 +3162,10 @@ public class TestReplicationScenarios { String dbName = createDBNonRepl(testName.getMethodName(), driver); verifyFail("REPL DUMP " + dbName, driver); verifyFail("REPL DUMP " + dbName + " from 1 ", driver); + assertTrue(run("REPL DUMP " + dbName + " with ('hive.repl.dump.metadata.only' = 'true')", + true, driver)); + assertTrue(run("REPL DUMP " + dbName + " from 1 with ('hive.repl.dump.metadata.only' = 'true')", + true, driver)); run("alter database " + dbName + " set dbproperties ('repl.source.for' = '1, 2, 3')", driver); assertTrue(run("REPL DUMP " + dbName, true, driver)); assertTrue(run("REPL DUMP " + dbName + " from 1 ", true, driver)); http://git-wip-us.apache.org/repos/asf/hive/blob/368d9cff/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 4b3a976..26e308c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -404,6 +404,42 @@ public class TestReplicationScenariosAcrossInstances { } @Test + public void testNonReplDBMetadataReplication() throws Throwable { + String dbName = primaryDbName + "_metadata"; + WarehouseInstance.Tuple tuple = primary + .run("create database " + dbName) + .run("use " + dbName) + .run("create table table1 (i int, j int)") + .run("create table table2 (a int, city string) partitioned by (country string)") + .run("create table table3 (i int, j int)") + .run("insert into table1 values (1,2)") + .dump(dbName, null, Arrays.asList("'hive.repl.dump.metadata.only'='true'")); + + replica.load(replicatedDbName, tuple.dumpLocation) + .run("use " + replicatedDbName) + .run("show tables") + .verifyResults(new String[]{"table1", "table2", "table3"}) + .run("select * from table1") + .verifyResults(Collections.emptyList()); + + tuple = primary + .run("use " + dbName) + .run("alter table table1 rename to renamed_table1") + .run("insert into table2 partition(country='india') values (1,'mumbai') ") + .run("create table table4 (i int, j int)") + .dump(dbName, tuple.lastReplicationId, Arrays.asList("'hive.repl.dump.metadata.only'='true'")); + + replica.load(replicatedDbName, tuple.dumpLocation) + .run("use " + replicatedDbName) + .run("show tables") + .verifyResults(new String[] { "renamed_table1", "table2", "table3", "table4" }) + .run("select * from renamed_table1") + .verifyResults(Collections.emptyList()) + .run("select * from table2") + .verifyResults(Collections.emptyList()); + } + + @Test public void testBootStrapDumpOfWarehouse() throws Throwable { String randomOne = RandomStringUtils.random(10, true, false); String randomTwo = RandomStringUtils.random(10, true, false); http://git-wip-us.apache.org/repos/asf/hive/blob/368d9cff/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 356a8c4..9753b5c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -134,20 +134,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { private void initReplDump(ASTNode ast) throws HiveException { int numChildren = ast.getChildCount(); + boolean isMetaDataOnly = false; dbNameOrPattern = PlanUtils.stripQuotes(ast.getChild(0).getText()); - for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { - Database database = db.getDatabase(dbName); - if (database != null) { - if (!ReplChangeManager.isSourceOfReplication(database)) { - throw new SemanticException("Cannot dump database " + dbNameOrPattern + - " as it is not a source of replication"); - } - } else { - throw new SemanticException("Cannot dump database " + dbNameOrPattern + " as it does not exist"); - } - } - // skip the first node, which is always required int currNode = 1; while (currNode < numChildren) { @@ -157,6 +146,10 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { if (null != replConfigs) { for (Map.Entry<String, String> config : replConfigs.entrySet()) { conf.set(config.getKey(), config.getValue()); + if ("hive.repl.dump.metadata.only".equalsIgnoreCase(config.getKey()) && + "true".equalsIgnoreCase(config.getValue())) { + isMetaDataOnly = true; + } } } } else if (ast.getChild(currNode).getType() == TOK_TABNAME) { @@ -187,6 +180,18 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // move to the next root node currNode++; } + + for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { + Database database = db.getDatabase(dbName); + if (database != null) { + if (!ReplChangeManager.isSourceOfReplication(database) && !isMetaDataOnly) { + throw new SemanticException("Cannot dump database " + dbName + + " as it is not a source of replication"); + } + } else { + throw new SemanticException("Cannot dump database " + dbName + " as it does not exist"); + } + } } // REPL DUMP