[CARBONDATA-1443] Schema storage has issue while converting table object to json string
Every table object converts to json and splits to small splits as there is limit in hive metastore of 4000. While splitting the json string code has problem so it could not get table and throws NPE This closes #1314 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/baca6f92 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/baca6f92 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/baca6f92 Branch: refs/heads/branch-1.2 Commit: baca6f925ad973864ff432dbaccd6a99df5ea363 Parents: bb0b347 Author: Ravindra Pesala <ravi.pes...@gmail.com> Authored: Fri Sep 1 10:51:04 2017 +0530 Committer: chenliang613 <chenliang...@apache.org> Committed: Fri Sep 1 14:25:00 2017 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 51 +++++++++++++++++--- .../carbondata/core/util/CarbonUtilTest.java | 47 +++++++++++++++++- 2 files changed, 91 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 67c1d81..8b6e44a 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -1803,10 +1803,31 @@ public final class CarbonUtil { return !(null == badRecordsLocation || badRecordsLocation.length() == 0); } + /** + * Converts Tableinfo object to json multi string objects of size 4000 + * @param tableInfo + * @param seperator separator between each string + * @param quote Quote to be used for string + * @param prefix Prefix to be added before generated string + * @return + */ public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator, String quote, String prefix) { Gson gson = new Gson(); String schemaString = gson.toJson(tableInfo); + return splitSchemaStringToMultiString(seperator, quote, prefix, schemaString); + } + + /** + * Converts Json String to multi string objects of size 4000 + * @param schemaString Json string + * @param seperator separator between each string + * @param quote Quote to be used for string + * @param prefix Prefix to be added before generated string + * @return + */ + public static String splitSchemaStringToMultiString(String seperator, String quote, + String prefix, String schemaString) { int schemaLen = schemaString.length(); int splitLen = 4000; int parts = schemaLen / splitLen; @@ -1817,10 +1838,12 @@ public final class CarbonUtil { new StringBuilder(prefix).append(quote).append("carbonSchemaPartsNo").append(quote) .append(seperator).append("'").append(parts).append("',"); int runningLen = 0; - int endLen = splitLen; + int endLen = schemaLen > splitLen ? splitLen : schemaLen; for (int i = 0; i < parts; i++) { if (i == parts - 1) { - endLen = schemaLen % splitLen; + if (schemaLen % splitLen > 0) { + endLen = schemaLen % splitLen; + } } builder.append(quote).append("carbonSchema").append(i).append(quote).append(seperator); builder.append("'").append(schemaString.substring(runningLen, runningLen + endLen)) @@ -1833,10 +1856,25 @@ public final class CarbonUtil { return builder.toString(); } + /** + * Converts Tableinfo object to json multi string objects of size 4000 and stored in map + * @param tableInfo + * @return + */ public static Map<String, String> convertToMultiStringMap(TableInfo tableInfo) { - Map<String, String> map = new HashMap<>(); Gson gson = new Gson(); String schemaString = gson.toJson(tableInfo); + return splitSchemaStringToMap(schemaString); + } + + /** + * Converts Json string to multi string objects of size 4000 and stored in map + * + * @param schemaString + * @return + */ + public static Map<String, String> splitSchemaStringToMap(String schemaString) { + Map<String, String> map = new HashMap<>(); int schemaLen = schemaString.length(); int splitLen = 4000; int parts = schemaLen / splitLen; @@ -1845,10 +1883,12 @@ public final class CarbonUtil { } map.put("carbonSchemaPartsNo", parts + ""); int runningLen = 0; - int endLen = splitLen; + int endLen = schemaLen > splitLen ? splitLen : schemaLen; for (int i = 0; i < parts; i++) { if (i == parts - 1) { - endLen = schemaLen % splitLen; + if (schemaLen % splitLen > 0) { + endLen = schemaLen % splitLen; + } } map.put("carbonSchema" + i, schemaString.substring(runningLen, runningLen + endLen)); runningLen += splitLen; @@ -1856,7 +1896,6 @@ public final class CarbonUtil { return map; } - public static TableInfo convertGsonToTableInfo(Map<String, String> properties) { Gson gson = new Gson(); String partsNo = properties.get("carbonSchemaPartsNo"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index 776059f..4a8483f 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -27,6 +27,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; @@ -1002,6 +1003,50 @@ public class CarbonUtilTest { } + @Test + public void testSplitSchemaStringToMapWithLessThanSplitLen() { + String schema = generateString(399); + Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema); + assert (map.size() == 2); + String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema); + assert (schemaString.length() > schema.length()); + } + + @Test + public void testSplitSchemaStringToMapWithEqualThanSplitLen() { + String schema = generateString(4000); + Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema); + assert (map.size() == 2); + String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema); + assert (schemaString.length() > schema.length()); + } + + @Test + public void testSplitSchemaStringToMapWithMoreThanSplitLen() { + String schema = generateString(7999); + Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema); + assert (map.size() == 3); + String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema); + assert (schemaString.length() > schema.length()); + } + + @Test + public void testSplitSchemaStringToMapWithMultiplesOfSplitLen() { + String schema = generateString(12000); + Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema); + assert (map.size() == 4); + String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema); + assert (schemaString.length() > schema.length()); + } + + private String generateString(int length) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < length; i++) { + builder.append("a"); + } + return builder.toString(); + } + private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk, FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) { int[] range; @@ -1013,7 +1058,7 @@ public class CarbonUtilTest { int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord); assertTrue(expectRangeIndex[0] <= index && index <= range[1]); } - + @AfterClass public static void testcleanUp() { new File("../core/src/test/resources/testFile.txt").deleteOnExit();
