Repository: hive Updated Branches: refs/heads/master 031cfa212 -> 599a74f8f
HIVE-17391 Compaction fails if there is an empty value in tblproperties (Steve Yeom via Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/599a74f8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/599a74f8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/599a74f8 Branch: refs/heads/master Commit: 599a74f8f28c088d49194960b3fe664650636f4c Parents: 031cfa2 Author: Eugene Koifman <ekoif...@hortonworks.com> Authored: Mon Oct 16 13:08:08 2017 -0700 Committer: Eugene Koifman <ekoif...@hortonworks.com> Committed: Mon Oct 16 13:08:08 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hive/common/StringableMap.java | 45 ++++++++++++++++---- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 17 ++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/599a74f8/common/src/java/org/apache/hadoop/hive/common/StringableMap.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/StringableMap.java b/common/src/java/org/apache/hadoop/hive/common/StringableMap.java index 8a93c0f..8896ce5 100644 --- a/common/src/java/org/apache/hadoop/hive/common/StringableMap.java +++ b/common/src/java/org/apache/hadoop/hive/common/StringableMap.java @@ -35,16 +35,36 @@ public class StringableMap extends HashMap<String, String> { int numElements = Integer.parseInt(parts[0]); s = parts[1]; for (int i = 0; i < numElements; i++) { + // Get the key String. parts = s.split(":", 2); - int len = Integer.parseInt(parts[0]); - String key = null; - if (len > 0) key = parts[1].substring(0, len); + int len = Integer.parseInt(parts[0]); + String key = ""; // Default is now an empty string. + + if (len > 0) key = parts[1].substring(0, len); + // Please check the toString() method of this class. + // null has -1 as length and empty String has 0. + else if (len < 0) { + key = null; + len = 0; // Set 0 to 'len' for null-valued key + // since only len exists for null-valued key from the given String "s". + } + + // Get the value String for the key parts = parts[1].substring(len).split(":", 2); len = Integer.parseInt(parts[0]); - String value = null; + String value = ""; + if (len > 0) value = parts[1].substring(0, len); - s = parts[1].substring(len); + else if (len < 0) { + value = null; + len = 0; // Set 0 to 'len' since only len exists. + } + + // Put the entry into the HashMap<String, String>. put(key, value); + + // Move to the next substring to process. + s = parts[1].substring(len); } } @@ -59,14 +79,21 @@ public class StringableMap extends HashMap<String, String> { buf.append(':'); if (size() > 0) { for (Map.Entry<String, String> entry : entrySet()) { - int length = (entry.getKey() == null) ? 0 : entry.getKey().length(); - buf.append(entry.getKey() == null ? 0 : length); + // Append the key String to the output StringBuilder. + // Note that null is saved as -1 in length, and that empty String as 0. + int length = (entry.getKey() == null) ? -1 : entry.getKey().length(); + buf.append(length); buf.append(':'); if (length > 0) buf.append(entry.getKey()); - length = (entry.getValue() == null) ? 0 : entry.getValue().length(); + + // Append the value String to the output StringBuilder. + // Note that null is saved as -1 in length, and that empty String as 0. + length = (entry.getValue() == null) ? -1 : entry.getValue().length(); buf.append(length); buf.append(':'); - if (length > 0) buf.append(entry.getValue()); + if (length > 0) { + buf.append(entry.getValue()); + } } } return buf.toString(); http://git-wip-us.apache.org/repos/asf/hive/blob/599a74f8/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index d99b3e3..3737b6a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -937,6 +937,23 @@ public class TestTxnCommands2 { } /** + * https://issues.apache.org/jira/browse/HIVE-17391 + */ + @Test + public void testEmptyInTblproperties() throws Exception { + runStatementOnDriver("create table t1 " + "(a int, b int) stored as orc TBLPROPERTIES ('serialization.null.format'='', 'transactional'='true')"); + runStatementOnDriver("insert into t1 " + "(a,b) values(1,7),(3,7)"); + runStatementOnDriver("update t1" + " set b = -2 where b = 2"); + runStatementOnDriver("alter table t1 " + " compact 'MAJOR'"); + runWorker(hiveConf); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize()); + Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); + Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local")); + } + + /** * https://issues.apache.org/jira/browse/HIVE-10151 */ @Test
