Repository: hive Updated Branches: refs/heads/master 0e0c98039 -> c8dccfdde
HIVE-17504: Skip ACID table for replication (Tao Li, reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c8dccfdd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c8dccfdd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c8dccfdd Branch: refs/heads/master Commit: c8dccfdde6dc41b474066dc5eb2b9078676b6022 Parents: 0e0c980 Author: Daniel Dai <da...@hortonworks.com> Authored: Tue Sep 12 12:49:21 2017 -0700 Committer: Daniel Dai <da...@hortonworks.com> Committed: Tue Sep 12 12:49:21 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 36 ++++++++++++++++++++ .../apache/hadoop/hive/ql/parse/EximUtil.java | 3 +- 2 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c8dccfdd/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index aae5a7c..40fc3fa 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -2936,6 +2936,42 @@ public class TestReplicationScenarios { verifyRun("SELECT max(a) from " + dbName + "_dupe.ptned2 where b=1", new String[]{"8"}, driverMirror); } + @Test + public void testSkipTables() throws IOException { + String testName = "skipTables"; + String dbName = createDB(testName, driver); + + // Create table + run("CREATE TABLE " + dbName + ".acid_table (key int, value int) PARTITIONED BY (load_date date) " + + "CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver); + verifyIfTableExist(dbName, "acid_table", metaStoreClient); + + // Bootstrap test + advanceDumpDir(); + run("REPL DUMP " + dbName, driver); + String replDumpLocn = getResult(0, 0,driver); + String replDumpId = getResult(0, 1, true, driver); + LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId); + run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror); + verifyIfTableNotExist(dbName + "_dupe", "acid_table", metaStoreClientMirror); + + // // Create another table for incremental repl verification + run("CREATE TABLE " + dbName + ".acid_table_incremental (key int, value int) PARTITIONED BY (load_date date) " + + "CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver); + verifyIfTableExist(dbName, "acid_table_incremental", metaStoreClient); + + // Perform REPL-DUMP/LOAD + advanceDumpDir(); + run("REPL DUMP " + dbName + " FROM " + replDumpId, driver); + String incrementalDumpLocn = getResult(0,0,driver); + String incrementalDumpId = getResult(0,1,true,driver); + LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); + run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); + printOutput(driverMirror); + run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror); + verifyIfTableNotExist(dbName + "_dupe", "acid_table_incremental", metaStoreClientMirror); + } + private static String createDB(String name, Driver myDriver) { LOG.info("Testing " + name); String dbName = name + "_" + tid; http://git-wip-us.apache.org/repos/asf/hive/blob/c8dccfdd/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index ece5495..95d394e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -420,7 +420,8 @@ public class EximUtil { } if (replicationSpec.isInReplicationScope()) { - return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative()); + return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative() || + (tableHandle.getParameters() != null && StringUtils.equals(tableHandle.getParameters().get("transactional"), "true"))); } if (tableHandle.isNonNative()) {
