Repository: hive Updated Branches: refs/heads/master 954f83284 -> d9ec721b4
HIVE-17887: Incremental REPL LOAD with Drop partition event on timestamp type partition column fails (Sankar Hariappan, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d9ec721b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d9ec721b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d9ec721b Branch: refs/heads/master Commit: d9ec721b4f1160efa7b1022fb0e1809c27a857df Parents: 954f832 Author: Sankar Hariappan <sank...@apache.org> Authored: Wed Oct 25 20:07:31 2017 +0530 Committer: Sankar Hariappan <sank...@apache.org> Committed: Wed Oct 25 20:07:31 2017 +0530 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 29 ++++++++++++++++++-- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 5 ++-- .../repl/load/message/DropPartitionHandler.java | 2 +- 3 files changed, 31 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d9ec721b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index c1aac83..765fa11 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -288,8 +288,8 @@ public class TestReplicationScenarios { verifyRun("SELECT * from " + replicatedDbName + ".unptned", unptn_data, driverMirror); verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=1", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=2", ptn_data_2, driverMirror); - verifyRun("SELECT a from " + dbName + ".ptned_empty", empty, driverMirror); - verifyRun("SELECT * from " + dbName + ".unptned_empty", empty, driverMirror); + verifyRun("SELECT a from " + replicatedDbName + ".ptned_empty", empty, driverMirror); + verifyRun("SELECT * from " + replicatedDbName + ".unptned_empty", empty, driverMirror); } @Test @@ -1890,6 +1890,31 @@ public class TestReplicationScenarios { } @Test + public void testDropPartitionEventWithPartitionOnTimestampColumn() throws IOException { + String testName = "dropPartitionEventWithPartitionOnTimestampColumn"; + String dbName = createDB(testName, driver); + run("CREATE TABLE " + dbName + ".ptned(a string) PARTITIONED BY (b timestamp)", driver); + + // Bootstrap dump/load + String replDbName = dbName + "_dupe"; + Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); + + String[] ptn_data = new String[] { "fifteen" }; + String ptnVal = "2017-10-24 00:00:00.0"; + run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver); + + // Replicate insert event and verify + Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + verifyRun("SELECT a from " + replDbName + ".ptned where (b=\"" + ptnVal + "\") ORDER BY a", ptn_data, driverMirror); + + run("ALTER TABLE " + dbName + ".ptned DROP PARTITION(b=\"" + ptnVal + "\")", driver); + + // Replicate drop partition event and verify + incrementalLoadAndVerify(dbName, incrDump.lastReplId, replDbName); + verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal)), metaStoreClientMirror); + } + + @Test public void testRenameTableWithCM() throws IOException { String testName = "renameTableWithCM"; LOG.info("Testing " + testName); http://git-wip-us.apache.org/repos/asf/hive/blob/d9ec721b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 65c9091..87f6e1d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4429,9 +4429,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable { // to the metastore to allow it to do drop a partition or not, depending on a Predicate on the // parameter key values. for (DropTableDesc.PartSpec partSpec : dropTbl.getPartSpecs()){ + List<Partition> partitions = new ArrayList<>(); try { - for (Partition p : Iterables.filter( - db.getPartitionsByFilter(tbl, partSpec.getPartSpec().getExprString()), + db.getPartitionsByExpr(tbl, partSpec.getPartSpec(), conf, partitions); + for (Partition p : Iterables.filter(partitions, replicationSpec.allowEventReplacementInto())){ db.dropPartition(tbl.getDbName(),tbl.getTableName(),p.getValues(),true); } http://git-wip-us.apache.org/repos/asf/hive/blob/d9ec721b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java index 5456416..a88916e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java @@ -92,7 +92,7 @@ public class DropPartitionHandler extends AbstractMessageHandler { PrimitiveTypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo(type); ExprNodeColumnDesc column = new ExprNodeColumnDesc(pti, key, null, true); ExprNodeGenericFuncDesc op = DDLSemanticAnalyzer.makeBinaryPredicate( - "=", column, new ExprNodeConstantDesc(pti, val)); + "=", column, new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, val)); expr = (expr == null) ? op : DDLSemanticAnalyzer.makeBinaryPredicate("and", expr, op); } if (expr != null) {
