Repository: incubator-carbondata Updated Branches: refs/heads/master 05f56afdc -> 00c479463
support octal escape delimited style style add testcase load csv with delimiter char \017 add testcase load csv with delimiter char \017 Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c1c9ac52 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c1c9ac52 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c1c9ac52 Branch: refs/heads/master Commit: c1c9ac529c026d10a2de168dedda98f7f302e44a Parents: 05f56af Author: boczhaow <pal...@sina.com> Authored: Tue Dec 13 11:06:54 2016 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Tue Dec 13 15:27:44 2016 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 17 +---------------- .../src/test/resources/sample_withDelimiter017.csv | 5 +++++ .../testsuite/dataload/TestLoadDataGeneral.scala | 9 +++++++++ 3 files changed, 15 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c1c9ac52/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 6531352..3980cb3 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -599,22 +599,7 @@ public final class CarbonUtil { * @return */ public static String unescapeChar(String parseStr) { - switch (parseStr) { - case "\\001": - return "\001"; - case "\\t": - return "\t"; - case "\\r": - return "\r"; - case "\\b": - return "\b"; - case "\\f": - return "\f"; - case "\\n": - return "\n"; - default: - return parseStr; - } + return scala.StringContext.treatEscapes(parseStr); } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c1c9ac52/integration/spark/src/test/resources/sample_withDelimiter017.csv ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/sample_withDelimiter017.csv b/integration/spark/src/test/resources/sample_withDelimiter017.csv new file mode 100644 index 0000000..c40b03a --- /dev/null +++ b/integration/spark/src/test/resources/sample_withDelimiter017.csv @@ -0,0 +1,5 @@ +idnamecityage +1davidshenzhen31 +2easonshenzhen27 +3jarrywuhan35 +3jarryBangalore35 http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c1c9ac52/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala index 7e102df..9904c93 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala @@ -79,6 +79,15 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll { ) } + test("test data loading CSV file with delimiter char \\017") { + val testData = currentDirectory + "/src/test/resources/sample_withDelimiter017.csv" + sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest options ('delimiter'='\\017')") + checkAnswer( + sql("SELECT COUNT(*) FROM loadtest"), + Seq(Row(20)) + ) + } + test("test data loading with invalid values for mesasures") { val testData = currentDirectory + "/src/test/resources/invalidMeasures.csv" sql("drop table if exists invalidMeasures")
