Repository: hive Updated Branches: refs/heads/master ad5bcb150 -> b4b06ac15
HIVE-17897: Repl load in bootstrap phase fails when partitions have whitespace (Thejas Nair, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4b06ac1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4b06ac1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4b06ac1 Branch: refs/heads/master Commit: b4b06ac1568490200f817cd7f1855b9a81df6bf9 Parents: ad5bcb1 Author: Sankar Hariappan <sank...@apache.org> Authored: Fri Dec 22 02:13:26 2017 +0530 Committer: Sankar Hariappan <sank...@apache.org> Committed: Fri Dec 22 02:13:45 2017 +0530 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 36 ++++++++++++++++++-- .../ql/parse/repl/dump/io/FileOperations.java | 2 +- 2 files changed, 35 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b4b06ac1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 9062d43..0e08007 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -1953,13 +1953,16 @@ public class TestReplicationScenarios { String testName = "dropPartitionEventWithPartitionOnTimestampColumn"; String dbName = createDB(testName, driver); run("CREATE TABLE " + dbName + ".ptned(a string) PARTITIONED BY (b timestamp)", driver); + String[] ptn_data = new String[] { "fourteen" }; + String ptnVal = "2017-10-01 01:00:10.1"; + run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver); // Bootstrap dump/load String replDbName = dbName + "_dupe"; Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String[] ptn_data = new String[] { "fifteen" }; - String ptnVal = "2017-10-24 00:00:00.0"; + ptn_data = new String[] { "fifteen" }; + ptnVal = "2017-10-24 00:00:00.0"; run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver); // Replicate insert event and verify @@ -1973,6 +1976,35 @@ public class TestReplicationScenarios { verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal)), metaStoreClientMirror); } + /** + * Verify replication when string partition column value has special chars + * @throws IOException + */ + @Test + public void testWithStringPartitionSpecialChars() throws IOException { + String testName = "testWithStringPartitionSpecialChars"; + String dbName = createDB(testName, driver); + run("CREATE TABLE " + dbName + ".ptned(v string) PARTITIONED BY (p string)", driver); + String[] ptn_data = new String[] { "fourteen", "fifteen" }; + String[] ptnVal = new String [] {"has a space, /, and \t tab", "another set of '#@ chars" }; + run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(p=\"" + ptnVal[0] +"\") values('" + ptn_data[0] + "')", driver); + + // Bootstrap dump/load + String replDbName = dbName + "_dupe"; + Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); + + run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(p=\"" + ptnVal[1] +"\") values('" + ptn_data[1] + "')", driver); + // Replicate insert event and verify + Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + verifyRun("SELECT p from " + replDbName + ".ptned ORDER BY p desc", ptnVal, driverMirror); + + run("ALTER TABLE " + dbName + ".ptned DROP PARTITION(p=\"" + ptnVal[0] + "\")", driver); + + // Replicate drop partition event and verify + incrementalLoadAndVerify(dbName, incrDump.lastReplId, replDbName); + verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal[0])), metaStoreClientMirror); + } + @Test public void testRenameTableWithCM() throws IOException { String testName = "renameTableWithCM"; http://git-wip-us.apache.org/repos/asf/hive/blob/b4b06ac1/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 2f636b6..4642012 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -108,6 +108,6 @@ public class FileOperations { private String encodedUri(FileStatus fileStatus) throws IOException { Path currentDataFilePath = fileStatus.getPath(); String checkSum = ReplChangeManager.checksumFor(currentDataFilePath, dataFileSystem); - return ReplChangeManager.encodeFileUri(currentDataFilePath.toUri().toString(), checkSum); + return ReplChangeManager.encodeFileUri(currentDataFilePath.toString(), checkSum); } }
