[CARBONDATA-3153] Complex delimiters change Changed the two Complex Delimiters used to '\001' and '\002'.
This closes #2979 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/82adc50e Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/82adc50e Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/82adc50e Branch: refs/heads/master Commit: 82adc50e7d01d04d8d613aa1d8e8ed7b546573c5 Parents: db08fe1 Author: manishnalla1994 <manish.nalla1...@gmail.com> Authored: Fri Dec 7 14:55:58 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Fri Dec 14 12:37:36 2018 +0530 ---------------------------------------------------------------------- conf/dataload.properties.template | 4 +- docs/dml-of-carbondata.md | 12 +- .../hadoop/api/CarbonTableOutputFormat.java | 2 +- .../complexType/TestAdaptiveComplexType.scala | 104 ++++---- .../TestAdaptiveEncodingForNullValues.scala | 22 +- .../complexType/TestCompactionComplexType.scala | 250 +++++++++---------- .../complexType/TestComplexDataType.scala | 63 ++--- .../TestComplexTypeWithBigArray.scala | 2 +- ...estLoadDataWithHiveSyntaxDefaultFormat.scala | 2 +- .../VarcharDataTypesBasicTestCase.scala | 4 +- .../sql/commands/UsingCarbondataSuite.scala | 2 +- .../datasource/SparkCarbonDataSourceTest.scala | 4 +- .../carbondata/CarbonDataSourceSuite.scala | 2 +- .../carbondata/TestStreamingTableOpName.scala | 19 +- .../TestStreamingTableWithRowParser.scala | 19 +- .../processing/loading/model/LoadOption.java | 4 +- .../sdk/file/CarbonWriterBuilder.java | 4 +- .../sdk/file/CSVCarbonWriterTest.java | 4 +- 18 files changed, 260 insertions(+), 263 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/conf/dataload.properties.template ---------------------------------------------------------------------- diff --git a/conf/dataload.properties.template b/conf/dataload.properties.template index 0b2f6cf..3b582b2 100644 --- a/conf/dataload.properties.template +++ b/conf/dataload.properties.template @@ -51,10 +51,10 @@ delimiter=, #all_dictionary_path= #complex column's level 1 delimiter -#complex_delimiter_level_1=\\$ +#complex_delimiter_level_1='\\\001' #complex column's level 2 delimiter -#complex_delimiter_level_2=\\: +#complex_delimiter_level_2='\\\002' #timestamp type column's data format #dateformat= http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/docs/dml-of-carbondata.md ---------------------------------------------------------------------- diff --git a/docs/dml-of-carbondata.md b/docs/dml-of-carbondata.md index 65654a4..d26cf19 100644 --- a/docs/dml-of-carbondata.md +++ b/docs/dml-of-carbondata.md @@ -132,18 +132,18 @@ CarbonData DML statements are documented here,which includes: - ##### COMPLEX_DELIMITER_LEVEL_1: - Split the complex type data column in a row (eg., a$b$c --> Array = {a,b,c}). + Split the complex type data column in a row (eg., a\001b\001c --> Array = {a,b,c}). ``` - OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='$') + OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\\\001') ``` - ##### COMPLEX_DELIMITER_LEVEL_2: - Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a:b$c:d --> Array> = {{a,b},{c,d}}). + Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a\002b\001c\002d --> Array> = {{a,b},{c,d}}). ``` - OPTIONS('COMPLEX_DELIMITER_LEVEL_2'=':') + OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\\\002') ``` - ##### ALL_DICTIONARY_PATH: @@ -212,8 +212,8 @@ CarbonData DML statements are documented here,which includes: 'FILEHEADER'='empno,empname,designation,doj,workgroupcategory, workgroupcategoryname,deptno,deptname,projectcode, projectjoindate,projectenddate,attendance,utilization,salary', - 'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='$', - 'COMPLEX_DELIMITER_LEVEL_2'=':', + 'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\\\001', + 'COMPLEX_DELIMITER_LEVEL_2'='\\\002', 'ALL_DICTIONARY_PATH'='/opt/alldictionary/data.dictionary', 'SINGLE_PASS'='TRUE') ``` http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java index 0bcd7e1..dbd2f0e 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java @@ -338,7 +338,7 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje SKIP_EMPTY_LINE, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE))); - String complexDelim = conf.get(COMPLEX_DELIMITERS, "$" + "," + ":"); + String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002"); String[] split = complexDelim.split(","); model.setComplexDelimiterLevel1(split[0]); if (split.length > 1) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala index 7fff15d..28edc77 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala @@ -46,9 +46,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$20:30:40')") - sql("insert into adaptive values(2,'600$abc$20:30:40')") - sql("insert into adaptive values(3,'600$abc$20:30:40')") + sql("insert into adaptive values(1,'500\001abc\00120\00230\00240')") + sql("insert into adaptive values(2,'600\001abc\00120\00230\00240')") + sql("insert into adaptive values(3,'600\001abc\00120\00230\00240')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), Row(2, Row(600, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), @@ -72,9 +72,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'700$abc$200:300:400')") - sql("insert into adaptive values(3,'800$abc$200:300:400')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'700\001abc\001200\002300\002400')") + sql("insert into adaptive values(3,'800\001abc\001200\002300\002400')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), @@ -98,9 +98,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -124,9 +124,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -139,8 +139,8 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:smallint,name:string," + "marks:array<smallint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'100$abc$20:30:40')") - sql("insert into adaptive values(2,'200$abc$30:40:50')") + sql("insert into adaptive values(1,'100\001abc\00120\00230\00240')") + sql("insert into adaptive values(2,'200\001abc\00130\00240\00250')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(100, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), Row(2, Row(200, "abc", mutable.WrappedArray.make(Array(30, 40, 50)))))) @@ -151,8 +151,8 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:smallint,name:string," + "marks:array<smallint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'8000$abc$300:400:500')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 400, 500)))))) @@ -163,7 +163,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1$abc$20:30:40')") + sql("insert into adaptive values(1,'1\001abc\00120\00230\00240')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))))) } @@ -173,8 +173,8 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'8000$abc$300:400:500')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 400, 500)))))) @@ -198,9 +198,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -239,9 +239,9 @@ trait TestAdaptiveComplexType extends QueryTest { "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" + " " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -253,7 +253,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.323$abc$2.2:3.3:4.4')") + sql("insert into adaptive values(1,'1.323\001abc\0012.2\0023.3\0024.4')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))))) sql("Drop table if exists adaptive") @@ -277,7 +277,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.323$abc$20.2:30.3:40.4')") + sql("insert into adaptive values(1,'1.323\001abc\00120.2\00230.3\00240.4')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))))) sql("Drop table if exists adaptive") @@ -301,7 +301,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'10.323$abc$20.2:30.3:500.423')") + sql("insert into adaptive values(1,'10.323\001abc\00120.2\00230.3\002500.423')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))))) sql("Drop table if exists adaptive") @@ -325,7 +325,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1000.323$abc$20.2:30.3:50000.423')") + sql("insert into adaptive values(1,'1000.323\001abc\00120.2\00230.3\00250000.423')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))))) sql("Drop table if exists adaptive") @@ -349,7 +349,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.797693134862315$abc$2.2:30.3:1.797693134862315')") + sql("insert into adaptive values(1,'1.797693134862315\001abc\0012.2\00230.3\0021.797693134862315')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1.797693134862315, @@ -363,7 +363,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:decimal(3,2),name:string>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'3.2$abc')") + sql("insert into adaptive values(1,'3.2\001abc')") sql("select * from adaptive").show(false) } @@ -372,7 +372,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<decimal>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$20.2:30.3:40.4')") + sql("insert into adaptive values(1,'abc\00120.2\00230.3\00240.4')") sql("select * from adaptive").show(false) } @@ -383,7 +383,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'2017/01/01 00:00:00$abc')") + sql("insert into adaptive values(1,'2017/01/01 00:00:00\001abc')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(Timestamp.valueOf("2017-01-01 00:00:00.0"), "abc")))) } @@ -395,7 +395,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<timestamp>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$2017/01/01:2018/01/01')") + sql("insert into adaptive values(1,'abc\0012017/01/01\0022018/01/01')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row("abc", @@ -409,7 +409,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<date>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$2017-01-01')") + sql("insert into adaptive values(1,'abc\0012017-01-01')") sql("select * from adaptive").show(false) } @@ -418,7 +418,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$20:30:40')") + sql("insert into adaptive values(1,'11111\001abc\00120\00230\00240')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))))) } @@ -428,7 +428,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$200:300:400')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))))) sql("Drop table if exists adaptive") @@ -462,9 +462,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -488,9 +488,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -502,9 +502,9 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000000')") sql("select * from adaptive").show(false) } @@ -513,7 +513,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11$abc$20:30:40')") + sql("insert into adaptive values(1,'11\001abc\00120\00230\00240')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))))) } @@ -523,7 +523,7 @@ trait TestAdaptiveComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$200:300:400')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))))) sql("Drop table if exists adaptive") @@ -546,7 +546,7 @@ trait TestAdaptiveComplexType extends QueryTest { "create table adaptive(roll int, student struct<id:boolean,name:string," + "marks:array<boolean>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'true$abc$false:true:false')") + sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false)))))) } @@ -557,12 +557,12 @@ trait TestAdaptiveComplexType extends QueryTest { "create table adaptive(array1 array<struct<double1:double,double2:double,double3:double>>) " + "stored by 'carbondata'") sql( - "insert into adaptive values('10.35:40000.35:1.7976931348623157$67890985.888:65.5656:200')," + - "('20.25:50000.25:4.945464565654656546546546324$10000000:300000:3000')") + "insert into adaptive values('10.35\00240000.35\0021.7976931348623157\00167890985.888\00265.5656\002200')," + + "('20.25\00250000.25\0024.945464565654656546546546324\00110000000\002300000\0023000')") checkExistence(sql("select * from adaptive"), true, "1.0E7,300000.0,3000.0") sql("Drop table if exists adaptive") sql("create table adaptive(struct_arr struct<array_db1:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values('5555555.9559:12345678991234567:3444.999')") + sql("insert into adaptive values('5555555.9559\00212345678991234567\0023444.999')") checkExistence(sql("select * from adaptive"), true, "5555555.9559, 1.2345678991234568E16, 3444.999") http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala index 528fb69..d0b1df8 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala @@ -53,7 +53,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -64,7 +64,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:smallint,name:string," + "marks:array<smallint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -75,7 +75,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -85,7 +85,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -96,7 +96,7 @@ class TestAdaptiveEncodingForNullValues "create table adaptive(roll int, student struct<id:decimal(3,2),name:string," + "marks:array<decimal>>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -108,7 +108,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'null$abc')") + sql("insert into adaptive values(1,'null\001abc')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc")))) } @@ -120,7 +120,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<timestamp>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$null:null:null')") + sql("insert into adaptive values(1,'abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -130,7 +130,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<date>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$null:null:null')") + sql("insert into adaptive values(1,'abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -140,7 +140,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -150,7 +150,7 @@ class TestAdaptiveEncodingForNullValues sql( "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } @@ -161,7 +161,7 @@ class TestAdaptiveEncodingForNullValues "create table adaptive(roll int, student struct<id:boolean,name:string," + "marks:array<boolean>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'null$abc$null:null:null')") + sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null)))))) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala index 6ff8a15..a353ec0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala @@ -46,10 +46,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$20:30:40')") - sql("insert into adaptive values(2,'600$abc$30:30:40')") - sql("insert into adaptive values(3,'700$abc$40:30:40')") - sql("insert into adaptive values(4,'800$abc$50:30:40')") + sql("insert into adaptive values(1,'500\001abc\00120\00230\00240')") + sql("insert into adaptive values(2,'600\001abc\00130\00230\00240')") + sql("insert into adaptive values(3,'700\001abc\00140\00230\00240')") + sql("insert into adaptive values(4,'800\001abc\00150\00230\00240')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), @@ -75,10 +75,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'600$abc$300:300:400')") - sql("insert into adaptive values(3,'700$abc$400:300:400')") - sql("insert into adaptive values(4,'800$abc$500:300:400')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'600\001abc\001300\002300\002400')") + sql("insert into adaptive values(3,'700\001abc\001400\002300\002400')") + sql("insert into adaptive values(4,'800\001abc\001500\002300\002400')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), @@ -104,10 +104,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:4000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:5000000:4000000')") - sql("insert into adaptive values(4,'200000$abc$2000000:6000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0024000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0025000000\0024000000')") + sql("insert into adaptive values(4,'200000\001abc\0012000000\0026000000\0024000000')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -133,10 +133,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$210:350:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") - sql("insert into adaptive values(4,'10000001$abd$250:450:62000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001210\002350\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(4,'10000001\001abd\001250\002450\00262000000')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -151,10 +151,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:smallint,name:string," + "marks:array<smallint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'100$abc$20:30:40')") - sql("insert into adaptive values(2,'200$abc$30:40:50')") - sql("insert into adaptive values(3,'300$abd$30:41:55')") - sql("insert into adaptive values(4,'400$abe$30:42:56')") + sql("insert into adaptive values(1,'100\001abc\00120\00230\00240')") + sql("insert into adaptive values(2,'200\001abc\00130\00240\00250')") + sql("insert into adaptive values(3,'300\001abd\00130\00241\00255')") + sql("insert into adaptive values(4,'400\001abe\00130\00242\00256')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(100, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), @@ -168,20 +168,20 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:smallint,name:string," + "marks:array<smallint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'8000$abc$300:410:500')") - sql("insert into adaptive values(3,'9000$abee$310:420:400')") - sql("insert into adaptive values(4,'9900$abfffffffffffffff$320:430:500')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'8000\001abc\001300\002410\002500')") + sql("insert into adaptive values(3,'9000\001abee\001310\002420\002400')") + sql("insert into adaptive values(4,'9900\001abfffffffffffffff\001320\002430\002500')") sql("alter table adaptive compact 'major'").show(200,false) checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 410, 500)))), Row(3, Row(9000, "abee", mutable.WrappedArray.make(Array(310, 420, 400)))), Row(4, Row(9900, "abfffffffffffffff", mutable.WrappedArray.make(Array(320, 430, 500)))))) - sql("insert into adaptive values(5,'500$abc$200:310:400')") - sql("insert into adaptive values(6,'8000$abc$300:310:500')") - sql("insert into adaptive values(7,'9000$abee$310:320:400')") - sql("insert into adaptive values(8,'9900$abfffffffffffffffeeee$320:330:500')") + sql("insert into adaptive values(5,'500\001abc\001200\002310\002400')") + sql("insert into adaptive values(6,'8000\001abc\001300\002310\002500')") + sql("insert into adaptive values(7,'9000\001abee\001310\002320\002400')") + sql("insert into adaptive values(8,'9900\001abfffffffffffffffeeee\001320\002330\002500')") sql("alter table adaptive compact 'major'").show(200,false) sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false) sql("clean files for table adaptive").show(200,false) @@ -202,18 +202,18 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(11,'1$abc$21:30:40')") - sql("insert into adaptive values(12,'1$ab1$22:30:40')") - sql("insert into adaptive values(13,'1$ab2$23:30:40')") - sql("insert into adaptive values(14,'1$ab3$24:30:40')") - sql("insert into adaptive values(15,'1$ab4$25:30:40')") - sql("insert into adaptive values(16,'1$ab5$26:30:40')") - sql("insert into adaptive values(17,'1$ab6$27:30:40')") - sql("insert into adaptive values(18,'1$ab7$28:30:40')") - sql("insert into adaptive values(19,'1$ab8$29:30:40')") - sql("insert into adaptive values(20,'1$ab9$30:30:40')") - sql("insert into adaptive values(21,'1$ab10$31:30:40')") - sql("insert into adaptive values(22,'1$ab11$32:30:40')") + sql("insert into adaptive values(11,'1\001abc\00121\00230\00240')") + sql("insert into adaptive values(12,'1\001ab1\00122\00230\00240')") + sql("insert into adaptive values(13,'1\001ab2\00123\00230\00240')") + sql("insert into adaptive values(14,'1\001ab3\00124\00230\00240')") + sql("insert into adaptive values(15,'1\001ab4\00125\00230\00240')") + sql("insert into adaptive values(16,'1\001ab5\00126\00230\00240')") + sql("insert into adaptive values(17,'1\001ab6\00127\00230\00240')") + sql("insert into adaptive values(18,'1\001ab7\00128\00230\00240')") + sql("insert into adaptive values(19,'1\001ab8\00129\00230\00240')") + sql("insert into adaptive values(20,'1\001ab9\00130\00230\00240')") + sql("insert into adaptive values(21,'1\001ab10\00131\00230\00240')") + sql("insert into adaptive values(22,'1\001ab11\00132\00230\00240')") sql("alter table adaptive compact 'major'").show(200,false) sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false) sql("clean files for table adaptive").show(200,false) @@ -240,10 +240,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'500$abc$200:300:400')") - sql("insert into adaptive values(2,'8000$abc$300:400:500')") - sql("insert into adaptive values(3,'9000$abc$300:400:500')") - sql("insert into adaptive values(4,'10000$abc$300:400:500')") + sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')") + sql("insert into adaptive values(3,'9000\001abc\001300\002400\002500')") + sql("insert into adaptive values(4,'10000\001abc\001300\002400\002500')") sql("alter table adaptive compact'major'") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), @@ -293,18 +293,18 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:bigint,name:string," + "marks:array<bigint>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") sql("alter table adaptive compact'major'") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -400,18 +400,18 @@ class TestCompactionComplexType extends QueryTest { "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" + " " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -434,10 +434,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.323$abc$2.2:3.3:4.4')") - sql("insert into adaptive values(2,'1.324$abc$2.2:3.3:4.4')") - sql("insert into adaptive values(3,'1.325$abc$2.2:3.3:4.4')") - sql("insert into adaptive values(4,'1.326$abc$2.2:3.3:4.4')") + sql("insert into adaptive values(1,'1.323\001abc\0012.2\0023.3\0024.4')") + sql("insert into adaptive values(2,'1.324\001abc\0012.2\0023.3\0024.4')") + sql("insert into adaptive values(3,'1.325\001abc\0012.2\0023.3\0024.4')") + sql("insert into adaptive values(4,'1.326\001abc\0012.2\0023.3\0024.4')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))), @@ -491,10 +491,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.323$abc$20.2:30.3:40.4')") - sql("insert into adaptive values(2,'1.324$abc$20.2:30.3:40.5')") - sql("insert into adaptive values(3,'1.325$abc$20.2:30.3:40.6')") - sql("insert into adaptive values(4,'1.326$abc$20.2:30.3:40.7')") + sql("insert into adaptive values(1,'1.323\001abc\00120.2\00230.3\00240.4')") + sql("insert into adaptive values(2,'1.324\001abc\00120.2\00230.3\00240.5')") + sql("insert into adaptive values(3,'1.325\001abc\00120.2\00230.3\00240.6')") + sql("insert into adaptive values(4,'1.326\001abc\00120.2\00230.3\00240.7')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))), @@ -549,10 +549,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'10.323$abc$20.2:30.3:501.423')") - sql("insert into adaptive values(2,'10.323$abc$20.2:30.3:502.421')") - sql("insert into adaptive values(3,'10.323$abc$20.2:30.3:503.422')") - sql("insert into adaptive values(4,'10.323$abc$20.2:30.3:504.424')") + sql("insert into adaptive values(1,'10.323\001abc\00120.2\00230.3\002501.423')") + sql("insert into adaptive values(2,'10.323\001abc\00120.2\00230.3\002502.421')") + sql("insert into adaptive values(3,'10.323\001abc\00120.2\00230.3\002503.422')") + sql("insert into adaptive values(4,'10.323\001abc\00120.2\00230.3\002504.424')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 501.423)))), @@ -606,10 +606,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1000.323$abc$20.2:30.3:60000.423')") - sql("insert into adaptive values(2,'1000.324$abc$20.2:30.3:70000.424')") - sql("insert into adaptive values(3,'1000.325$abc$20.2:30.3:80000.425')") - sql("insert into adaptive values(4,'1000.326$abc$20.2:30.3:90000.426')") + sql("insert into adaptive values(1,'1000.323\001abc\00120.2\00230.3\00260000.423')") + sql("insert into adaptive values(2,'1000.324\001abc\00120.2\00230.3\00270000.424')") + sql("insert into adaptive values(3,'1000.325\001abc\00120.2\00230.3\00280000.425')") + sql("insert into adaptive values(4,'1000.326\001abc\00120.2\00230.3\00290000.426')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(1000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 60000.423)))), @@ -664,10 +664,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:double,name:string," + "marks:array<double>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'1.797693134862315$abc$2.2:30.3:1.797693134862315')") - sql("insert into adaptive values(2,'1.797693134862316$abc$2.2:30.3:1.797693134862316')") - sql("insert into adaptive values(3,'1.797693134862317$abc$2.2:30.3:1.797693134862317')") - sql("insert into adaptive values(4,'1.797693134862318$abc$2.2:30.3:1.797693134862318')") + sql("insert into adaptive values(1,'1.797693134862315\001abc\0012.2\00230.3\0021.797693134862315')") + sql("insert into adaptive values(2,'1.797693134862316\001abc\0012.2\00230.3\0021.797693134862316')") + sql("insert into adaptive values(3,'1.797693134862317\001abc\0012.2\00230.3\0021.797693134862317')") + sql("insert into adaptive values(4,'1.797693134862318\001abc\0012.2\00230.3\0021.797693134862318')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, @@ -695,7 +695,7 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:decimal(3,2),name:string>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'3.2$abc')") + sql("insert into adaptive values(1,'3.2\001abc')") sql("select * from adaptive").show(false) } @@ -704,7 +704,7 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<decimal>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$20.2:30.3:40.4')") + sql("insert into adaptive values(1,'abc\00120.2\00230.3\00240.4')") sql("select * from adaptive").show(false) } @@ -715,10 +715,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " + "'carbondata'") - sql("insert into adaptive values(1,'2017/01/01 00:00:00$abc')") - sql("insert into adaptive values(2,'2017/01/02 00:00:00$abc')") - sql("insert into adaptive values(3,'2017/01/03 00:00:00$abc')") - sql("insert into adaptive values(4,'2017/01/04 00:00:00$abc')") + sql("insert into adaptive values(1,'2017/01/01 00:00:00\001abc')") + sql("insert into adaptive values(2,'2017/01/02 00:00:00\001abc')") + sql("insert into adaptive values(3,'2017/01/03 00:00:00\001abc')") + sql("insert into adaptive values(4,'2017/01/04 00:00:00\001abc')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(Timestamp.valueOf("2017-01-01 00:00:00.0"), "abc")), @@ -735,10 +735,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<timestamp>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc1$2017/01/01:2018/01/01')") - sql("insert into adaptive values(2,'abc2$2017/01/02:2018/01/03')") - sql("insert into adaptive values(3,'abc3$2017/01/04:2018/01/05')") - sql("insert into adaptive values(4,'abc4$2017/01/06:2018/01/07')") + sql("insert into adaptive values(1,'abc1\0012017/01/01\0022018/01/01')") + sql("insert into adaptive values(2,'abc2\0012017/01/02\0022018/01/03')") + sql("insert into adaptive values(3,'abc3\0012017/01/04\0022018/01/05')") + sql("insert into adaptive values(4,'abc4\0012017/01/06\0022018/01/07')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, @@ -769,7 +769,7 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<name:string," + "marks:array<date>>) stored by 'carbondata'") - sql("insert into adaptive values(1,'abc$2017-01-01')") + sql("insert into adaptive values(1,'abc\0012017-01-01')") sql("select * from adaptive").show(false) } @@ -778,10 +778,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$20:30:40')") - sql("insert into adaptive values(2,'11111$abc$55:65:75')") - sql("insert into adaptive values(3,'11111$abc$88:98:8')") - sql("insert into adaptive values(4,'11111$abc$99:9:19')") + sql("insert into adaptive values(1,'11111\001abc\00120\00230\00240')") + sql("insert into adaptive values(2,'11111\001abc\00155\00265\00275')") + sql("insert into adaptive values(3,'11111\001abc\00188\00298\0028')") + sql("insert into adaptive values(4,'11111\001abc\00199\0029\00219')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))), @@ -796,10 +796,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$200:300:400')") - sql("insert into adaptive values(2,'11111$abc$201:301:401')") - sql("insert into adaptive values(3,'11111$abc$202:302:402')") - sql("insert into adaptive values(4,'11111$abc$203:303:403')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')") + sql("insert into adaptive values(2,'11111\001abc\001201\002301\002401')") + sql("insert into adaptive values(3,'11111\001abc\001202\002302\002402')") + sql("insert into adaptive values(4,'11111\001abc\001203\002303\002403')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), @@ -861,9 +861,9 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')") - sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')") + sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')") + sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))), @@ -887,9 +887,9 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))), @@ -901,9 +901,9 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'500000$abc$200:300:52000000000')") - sql("insert into adaptive values(2,'700000$abc$200:300:52000000000')") - sql("insert into adaptive values(3,'10000000$abc$200:300:52000000000')") + sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000000')") + sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000000')") + sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000000')") sql("select * from adaptive").show(false) } @@ -912,7 +912,7 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11$abc$20:30:40')") + sql("insert into adaptive values(1,'11\001abc\00120\00230\00240')") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))))) } @@ -922,10 +922,10 @@ class TestCompactionComplexType extends QueryTest { sql( "create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'11111$abc$200:300:400')") - sql("insert into adaptive values(1,'11111$abc$200:300:401')") - sql("insert into adaptive values(1,'11111$abc$200:300:402')") - sql("insert into adaptive values(1,'11111$abc$200:300:403')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002401')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002402')") + sql("insert into adaptive values(1,'11111\001abc\001200\002300\002403')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))), @@ -976,10 +976,10 @@ class TestCompactionComplexType extends QueryTest { "create table adaptive(roll int, student struct<id:boolean,name:string," + "marks:array<boolean>>) " + "stored by 'carbondata'") - sql("insert into adaptive values(1,'true$abc$false:true:false')") - sql("insert into adaptive values(1,'true$abc$false:true:true')") - sql("insert into adaptive values(1,'true$abc$false:true:true')") - sql("insert into adaptive values(1,'true$abc$false:true:false')") + sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')") + sql("insert into adaptive values(1,'true\001abc\001false\002true\002true')") + sql("insert into adaptive values(1,'true\001abc\001false\002true\002true')") + sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')") sql("alter table adaptive compact 'major' ") checkAnswer(sql("select * from adaptive"), Seq(Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false)))), http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala index f2e33f3..40ff648 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala @@ -58,7 +58,8 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll string,person array<int>) stored by " + "'carbondata'") - sql("insert into table1 values('abc','1$2$3')") + sql("insert into table1 values('abc','1\0012\0013')") + sql("select * from table1").show(false) checkAnswer(sql("select roll,person from table1"), Seq(Row("abc", mutable.WrappedArray.make(Array(1, 2, 3))))) } @@ -68,7 +69,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<int>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'1:2')") + sql("insert into table1 values(1,'1\0022')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(1))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(2))) checkAnswer(sql("select roll,person from table1"), @@ -96,7 +97,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<string>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'abc:bcd')") + sql("insert into table1 values(1,'abc\002bcd')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row("abc"))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row("bcd"))) checkAnswer(sql("select roll,person from table1"), @@ -122,7 +123,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<double>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'10.00:20.00')") + sql("insert into table1 values(1,'10.00\00220.00')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(10.0))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(20.0))) checkAnswer(sql("select roll,person from table1"), @@ -145,7 +146,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<decimal(3,2)>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'3.4:4.2')") + sql("insert into table1 values(1,'3.4\0024.2')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(3.40))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(4.20))) checkAnswer(sql("select roll,person.detail[0] from table1"), Seq(Row(1, 3.40))) @@ -178,7 +179,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<timestamp>>) stored by " + "'carbondata'") - sql("insert into table1 select 1,'2018/01/01:2017/01/01'") + sql("insert into table1 select 1,'2018/01/01\0022017/01/01'") checkExistence(sql("select person.detail[0] from table1"), true, "2018-01-01 00:00:00.0") checkExistence(sql("select person.detail[1] from table1"), true, "2017-01-01 00:00:00.0") checkAnswer(sql("select roll,person from table1"), @@ -208,7 +209,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<long>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'2018888:2018889')") + sql("insert into table1 values(1,'2018888\0022018889')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(2018888))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(2018889))) checkAnswer(sql("select person,roll from table1"), @@ -232,7 +233,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<short>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'20:30')") + sql("insert into table1 values(1,'20\00230')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(20))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(30))) checkAnswer(sql("select person,roll from table1"), @@ -256,7 +257,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (roll int,person Struct<detail:array<boolean>>) stored by " + "'carbondata'") - sql("insert into table1 values(1,'true:false')") + sql("insert into table1 values(1,'true\002false')") checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(true))) checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(false))) checkAnswer(sql("select person,roll from table1"), @@ -374,7 +375,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table table1 (person Struct<detail:string,ph:array<int>>) stored by " + "'carbondata' tblproperties('dictionary_include'='person')") - sql("insert into table1 values ('abc$2')") + sql("insert into table1 values ('abc\0012')") sql("select person from table1").show(false) sql("select person.detail, person.ph[0] from table1").show(false) } @@ -518,14 +519,14 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "create table table1 (roll string,person Struct<detail:int,age:string>,person1 " + "Struct<detail:int,age:array<string>>) stored by " + "'carbondata'") - sql("insert into table1 values('abc','1$abc','2$cde')") + sql("insert into table1 values('abc','1\001abc','2\001cde')") sql("select person.detail,person1.age from table1").show(false) } test("test Projection PushDown for more than one Struct column Cases -1") { sql("drop table if exists test") sql("create table test (a struct<b:int, c:struct<d:int,e:int>>) stored by 'carbondata'") - sql("insert into test select '1$2:3'") + sql("insert into test select '1\0012\0023'") checkAnswer(sql("select * from test"), Seq(Row(Row(1, Row(2, 3))))) checkAnswer(sql("select a.b,a.c from test"), Seq(Row(1, Row(2, 3)))) checkAnswer(sql("select a.c, a.b from test"), Seq(Row(Row(2, 3), 1))) @@ -552,9 +553,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "h:string,i:int>,j:int>) stored " + "by " + "'carbondata'") - sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')") - sql("insert into table1 values(2,'1$abc$2$efg$3:mno:4$5')") - sql("insert into table1 values(3,'1$abc$2$efg$3:mno:4$5')") + sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')") + sql("insert into table1 values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')") + sql("insert into table1 values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')") checkAnswer(sql("select a.b from table1"), Seq(Row(1), Row(1), Row(1))) checkAnswer(sql("select a.c from table1"), Seq(Row("abc"), Row("abc"), Row("abc"))) checkAnswer(sql("select a.d from table1"), Seq(Row(2), Row(2), Row(2))) @@ -596,9 +597,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "h:string,i:int>,j:int>) stored " + "by " + "'carbondata' tblproperties('dictionary_include'='a')") - sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')") - sql("insert into table1 values(2,'1$abc$2$efg$3:mno:4$5')") - sql("insert into table1 values(3,'1$abc$2$efg$3:mno:4$5')") + sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')") + sql("insert into table1 values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')") + sql("insert into table1 values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')") checkAnswer(sql("select a.b from table1"), Seq(Row(1), Row(1), Row(1))) checkAnswer(sql("select a.c from table1"), Seq(Row("abc"), Row("abc"), Row("abc"))) @@ -656,7 +657,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "create table test(cus_id string, struct_of_array struct<id:int,date:timestamp," + "sno:array<int>,sal:array<double>,state:array<string>,date1:array<timestamp>>) stored by " + "'carbondata'") - sql("insert into test values('cus_01','1$2017/01/01$1:2$2.0:3.0$ab:ac$2018/01/01')") + sql("insert into test values('cus_01','1\0012017/01/01\0011\0022\0012.0\0023.0\001ab\002ac\0012018/01/01')") // sql("select *from test").show(false) sql( "select struct_of_array.state[0],count(distinct struct_of_array.id) as count_int,count" + @@ -672,7 +673,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS test") sql("create table test(cus_id string,array_of_struct array<struct<id:int,country:string," + "state:string,city:string>>) stored by 'carbondata'") - sql("insert into test values('cus_01','123:abc:mno:xyz$1234:abc1:mno1:xyz1')") + sql("insert into test values('cus_01','123\002abc\002mno\002xyz\0011234\002abc1\002mno1\002xyz1')") checkAnswer(sql("select array_of_struct.state[0],count(distinct array_of_struct.id[0]) as count_country," + "count(distinct array_of_struct.state[0]) as count_city from test group by array_of_struct" + ".state[0]"), Seq(Row("mno", 1, 1))) @@ -681,9 +682,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { test("test struct complex type with filter") { sql("DROP TABLE IF EXISTS test") sql("create table test(id int,a struct<b:int,c:int>) stored by 'carbondata'") - sql("insert into test values(1,'2$3')") - sql("insert into test values(3,'5$3')") - sql("insert into test values(2,'4$5')") + sql("insert into test values(1,'2\0013')") + sql("insert into test values(3,'5\0013')") + sql("insert into test values(2,'4\0015')") checkAnswer(sql("select a.b from test where id=3"),Seq(Row(5))) checkAnswer(sql("select a.b from test where a.c!=3"),Seq(Row(4))) checkAnswer(sql("select a.b from test where a.c=3"),Seq(Row(5),Row(2))) @@ -710,7 +711,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { test("test Projection with two struct") { sql("DROP TABLE IF EXISTS test") sql("create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>) stored by 'carbondata'") - sql("insert into test values(1,'2$3','3$2')") + sql("insert into test values(1,'2\0013','3\0012')") checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(3,2)))) checkAnswer(sql("select a.b,id,a.c from test"),Seq(Row(2,1,3))) checkAnswer(sql("select d.e,d.f from test"),Seq(Row(3,2))) @@ -730,7 +731,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { test("test project with struct and array") { sql("DROP TABLE IF EXISTS test") sql("create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>,person Struct<detail:array<int>>) stored by 'carbondata'") - sql("insert into test values(1,'2$3','3$2','5:6:7:8')") + sql("insert into test values(1,'2\0013','3\0012','5\0026\0027\0028')") checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(3,2),Row(mutable.WrappedArray.make(Array(5,6,7,8)))))) checkAnswer(sql("select a.b,id,a.c,person.detail[0] from test"),Seq(Row(2,1,3,5))) checkAnswer(sql("select a.b,id,a.c,person.detail[0],d.e,d.f,person.detail[1],id from test"),Seq(Row(2,1,3,5,3,2,6,1))) @@ -740,7 +741,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { test("test block Update for complex datatype") { sql("DROP TABLE IF EXISTS test") sql("create table test(id int,a struct<b:int,c:int>,d array<int>) stored by 'carbondata'") - sql("insert into test values(1,'2$3',4)") + sql("insert into test values(1,'2\0013',4)") val structException = intercept[UnsupportedOperationException]( sql("update test set(a.b)=(4) where id=1").show(false)) assertResult("Unsupported operation on Complex data type")(structException.getMessage) @@ -809,7 +810,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "h:string,i:int>,j:int>) stored " + "by " + "'carbondata' tblproperties('dictionary_exclude'='a')") - sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')") + sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')") checkAnswer(sql("select a.b from table1"), Seq(Row(1))) sql("DROP TABLE IF EXISTS table1") val structException = intercept[MalformedCarbonCommandException]( @@ -872,7 +873,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS table1") sql( "create table table1 (person struct<b:array<double>>) stored by 'carbondata'") - sql("insert into table1 values('10000000:2000000000:2900000000')") + sql("insert into table1 values('10000000\0022000000000\0022900000000')") checkExistence(sql("select * from table1"),true,"2.9E9") } @@ -924,7 +925,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql( "create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>, d1 struct<e1:int," + "f1:int>) stored by 'carbondata' tblproperties('dictionary_include'='d1')") - sql("insert into test values(1,'2$3','4$5','6$7')") + sql("insert into test values(1,'2\0013','4\0015','6\0017')") checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(4,5),Row(6,7)))) sql("DROP TABLE IF EXISTS test") sql( @@ -971,7 +972,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { "MM-dd-yyyy") sql("DROP TABLE IF EXISTS test") sql("create table test(a struct<d1:date,d2:date>) stored by 'carbondata'") - sql("insert into test values ('02-18-2012$12-9-2016')") + sql("insert into test values ('02-18-2012\00112-9-2016')") checkAnswer(sql("select * from test "), Row(Row(java.sql.Date.valueOf("2012-02-18"),java.sql.Date.valueOf("2016-12-09")))) CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, @@ -981,7 +982,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS table1") sql( "create table table1 (id int, name string, structField struct<intval:int, stringval:string>) stored by 'carbondata'") - sql("insert into table1 values(null,'aaa','23$bb')") + sql("insert into table1 values(null,'aaa','23\001bb')") checkAnswer(sql("select * from table1"),Seq(Row(null,"aaa", Row(23,"bb")))) checkAnswer(sql("select id,name,structField.intval,structField.stringval from table1"),Seq(Row(null,"aaa",23,"bb"))) checkAnswer(sql("select id,name,structField.intval,structField.stringval,name from table1"),Seq(Row(null,"aaa",23,"bb","aaa"))) http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala index f4fd168..ac793ab 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala @@ -33,7 +33,7 @@ class TestComplexTypeWithBigArray extends QueryTest with BeforeAndAfterAll { override def beforeAll: Unit = { // write a CSV containing 32000 row, each row has an array with 10 elements val out = new PrintStream(new FileOutputStream(file)) - (1 to 33000).foreach(i=>out.println(s"$i,$i$$1")) + (1 to 33000).foreach(i=>out.println(s"$i,$i\0011")) out.close() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala index 7f150be..61271e1 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala @@ -373,7 +373,7 @@ class TestLoadDataWithHiveSyntaxDefaultFormat extends QueryTest with BeforeAndAf "('dictionary_include'='date1,date2')") CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") - sql("insert into array_timestamp values('2015/01/01$2016/01/01','2017/01/01')") + sql("insert into array_timestamp values('2015/01/01\0012016/01/01','2017/01/01')") checkExistence(sql("select * from array_timestamp "), true, "2015-01-01 00:00:00.0, 2016-01-01 00:00:00.0") checkExistence(sql("select * from array_timestamp "), http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala index 4051de4..a96f7df 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala @@ -331,8 +331,8 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi sql( """ | INSERT INTO TABLE varchar_complex_table - | VALUES(1,'ar1.0$ar1.1','longstr10','normal string1','longstr11','ar2.0$ar2.1'), - | (2,'ar1.2$ar1.3','longstr20','normal string2','longstr21','ar2.2$ar2.3') + | VALUES(1,'ar1.0\001ar1.1','longstr10','normal string1','longstr11','ar2.0\001ar2.1'), + | (2,'ar1.2\001ar1.3','longstr20','normal string2','longstr21','ar2.2\001ar2.3') | """.stripMargin) checkAnswer( sql("SELECT * FROM varchar_complex_table where varchar1='longstr10'"), http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala index 74e04b0..eafbf36 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala @@ -44,7 +44,7 @@ class UsingCarbondataSuite extends QueryTest with BeforeAndAfterEach { sql("DROP TABLE IF EXISTS create_source") sql("CREATE TABLE create_source(intField INT, stringField STRING, complexField ARRAY<INT>) " + "USING carbondata") - sql("""INSERT INTO create_source VALUES(1,"source","1$2$3")""") + sql("""INSERT INTO create_source VALUES(1,"source","1\0012\0013")""") checkAnswer(sql("SELECT * FROM create_source"), Row(1, "source", mutable.WrappedArray.newBuilder[Int].+=(1, 2, 3))) sql("DROP TABLE IF EXISTS create_source") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala index 470e0bf..026c5ca 100644 --- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala +++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala @@ -994,7 +994,7 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll { var i = 0 while (i < 11) { - val array = Array[String](s"name$i", s"$i" + "$" +s"$i.${i}12") + val array = Array[String](s"name$i", s"$i" + "\001" +s"$i.${i}12") writer.write(array) i += 1 } @@ -1124,7 +1124,7 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll { var i = 0 while (i < 10) { - val array = Array[String](s"name$i",s"$i" + "$" + s"${i*2}", s"${i/2}" + "$" + s"${i/3}") + val array = Array[String](s"name$i",s"$i" + "\001" + s"${i*2}", s"${i/2}" + "\001" + s"${i/3}") writer.write(array) i += 1 } http://git-wip-us.apache.org/repos/asf/carbondata/blob/82adc50e/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala index 7c1265c..c7c0d2c 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala @@ -251,7 +251,7 @@ class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll { sql("drop table if exists create_source") sql("create table create_source(intField int, stringField string, complexField array<int>) " + "USING org.apache.spark.sql.CarbonSource OPTIONS('bucketnumber'='1', 'bucketcolumns'='stringField', 'tableName'='create_source')") - sql("""insert into create_source values(1,"source","1$2$3")""") + sql("""insert into create_source values(1,"source","1\0012\0013")""") checkAnswer(sql("select * from create_source"), Row(1,"source", mutable.WrappedArray.newBuilder[Int].+=(1,2,3))) sql("drop table if exists create_source") }