jack86596 commented on a change in pull request #3664: [CARBONDATA-3740] Add line separator option to load command to configure the line separator during csv parsing. URL: https://github.com/apache/carbondata/pull/3664#discussion_r401340559
########## File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala ########## @@ -77,4 +87,54 @@ class TestLoadOptions extends QueryTest with BeforeAndAfterAll{ Row(1, "2015/7/23", "ind", "aaa1", "phone197", "ASD69643a", 15000)) } + test("test load data with line separator option value as Linux/Unix \"\\n\"") { + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/carriage_return_in_string.csv' INTO TABLE + |carriage_return_in_string OPTIONS ('fileheader'='id, name, city', 'line_separator'='\\n')""" + .stripMargin.replace('\n', ' ')); + checkAnswer(sql("select * from carriage_return_in_string where id = 1"), + Row(1, "2\r", "3")) + } + + test("test load data without line separator option") { + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/carriage_return_in_string.csv' INTO TABLE + |carriage_return_in_string OPTIONS ('fileheader'='id, name, city')""" + .stripMargin.replace('\n', ' ')); + checkAnswer(sql("select * from carriage_return_in_string where id = 1"), + Row(1, "2", null)) + } + + test("test load data with line separator option value as Windows \"\\r\\n\"") { + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/carriage_return_in_string.csv' INTO TABLE + |carriage_return_in_string OPTIONS ('fileheader'='id, name, city', + |'line_separator'='\\r\\n')""".stripMargin.replace('\n', ' ')); + checkAnswer(sql("select * from carriage_return_in_string where id = 1"), + Row(1, "2\r", "3\n4")) + } + + test("test load data with line separator option value as any two characters \"ab\"") { + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/carriage_return_in_string.csv' INTO TABLE + |carriage_return_in_string OPTIONS ('fileheader'='id, name, city', + |'line_separator'='ab')""".stripMargin.replace('\n', ' ')); + checkAnswer(sql("select * from carriage_return_in_string where id = 1"), + Row(1, "2\r", "3\n4")) + } + + test("test load data with line separator option value as empty") { + val exception = intercept[MalformedCarbonCommandException] { + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/carriage_return_in_string.csv' INTO TABLE + |carriage_return_in_string OPTIONS ('fileheader'='id, name, city', + |'line_separator'='')""".stripMargin.replace('\n', ' ')); + } + assert(exception.getMessage.contains("LINE_SEPARATOR can be only one or two characters.")) + } + + test("test load data with line separator option value as more then two characters" + Review comment: done ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services