[CARBONDATA-1417]Added cluster tests for IUD, batch sort and global sort features
This closes #1293 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fc39b287 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fc39b287 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fc39b287 Branch: refs/heads/branch-1.2 Commit: fc39b287a17a8b755f3a52b2b233d5cd6c7cdfcb Parents: 33ecca9 Author: Raghunandan S <carbondatacontributi...@gmail.com> Authored: Mon Aug 28 21:57:20 2017 +0530 Committer: Ravindra Pesala <ravi.pes...@gmail.com> Committed: Tue Sep 12 15:53:13 2017 +0530 ---------------------------------------------------------------------- .../src/test/resources/testdatafileslist.txt | 3 + .../sdv/generated/BatchSortLoad3TestCase.scala | 209 + .../sdv/generated/DataLoadingIUDTestCase.scala | 3773 ++++++++++++++++++ .../sdv/generated/GlobalSortTestCase.scala | 621 +++ .../cluster/sdv/suite/SDVSuites.scala | 5 +- 5 files changed, 4610 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc39b287/integration/spark-common-cluster-test/src/test/resources/testdatafileslist.txt ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/resources/testdatafileslist.txt b/integration/spark-common-cluster-test/src/test/resources/testdatafileslist.txt index 924756e..08ff519 100644 --- a/integration/spark-common-cluster-test/src/test/resources/testdatafileslist.txt +++ b/integration/spark-common-cluster-test/src/test/resources/testdatafileslist.txt @@ -98,6 +98,9 @@ Data/InsertData/WithHeaders.csv Data/InsertData/WithoutHeader.csv Data/uniqdata/1lac_UniqData.csv Data/batchsort/data.csv +Data/batchsort/1000_UniqData.csv +Data/batchsort/folder1/7000_UniqData.csv +Data/batchsort/folder1/folder2/1lac_UniqData.csv Data/3Lakh.csv Data/1lakh.csv Data/noinverted.csv http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc39b287/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala new file mode 100644 index 0000000..61e5bb7 --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad3TestCase.scala @@ -0,0 +1,209 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.cluster.sdv.generated + +import org.apache.spark.sql.common.util._ +import org.scalatest.BeforeAndAfterAll + +/** + * Test Class for BatchSortLoad3TestCase to verify all scenerios + */ + +class BatchSortLoad3TestCase extends QueryTest with BeforeAndAfterAll { + + override def beforeAll(): Unit = { + sql(s"""drop table if exists uniqdata20c""").collect + sql(s"""drop table if exists uniqdata19c""").collect + } +//Batch_sort_Loading_001-01-01-01_001-TC_020 + test("Batch_sort_Loading_001-01-01-01_001-TC_020", Include) { + sql( + s"""CREATE TABLE uniqdata20c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, + |DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint, + |DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, + |Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata' + | TBLPROPERTIES('NO_INVERTED_INDEX'='CUST_NAME')""".stripMargin.replaceAll(System + .lineSeparator, "")).collect + sql( + s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into + | table uniqdata20c OPTIONS('DELIMITER'=',' ,'QUOTECHAR'='"','COMMENTCHAR'='#', + | 'MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT', + | 'BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION, + | DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, + | Double_COLUMN2,INTEGER_COLUMN1')""".stripMargin.replaceAll(System + .lineSeparator, "")).collect + + sql(s"""select * from uniqdata20c""").collect + sql(s"""drop table if exists uniqdata20c""").collect + + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_046 + test("Batch_sort_Loading_001-01-01-01_001-TC_046", Include) { + sql( + s"""CREATE TABLE uniqdata19c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, + DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint, + DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, + Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""".stripMargin + .replaceAll(System.lineSeparator, "")).collect + + sql( + s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table + | uniqdata19c OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','COMMENTCHAR'='#', + | 'MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='IGNORE', + | 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1, + | BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, + | Double_COLUMN2,INTEGER_COLUMN1')""".stripMargin.replaceAll(System.lineSeparator, "")).collect + + sql(s"""select * from uniqdata19c""").collect + sql(s"""drop table if exists uniqdata19c""").collect + + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_053 + test("Batch_sort_Loading_001-01-01-01_001-TC_053", Include) { + sql( + s"""drop table if exists t3""").collect + sql( + s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype String, + |serialname String,salary Int,floatField float) + | STORED BY 'carbondata'""".stripMargin.replaceAll(System.lineSeparator, "")).collect + + sql( + s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' into table t3 options( + |'COLUMNDICT'='country:$resourcesPath/Data/columndict/country.csv','single_pass'='true')""" + .stripMargin.replaceAll(System.lineSeparator, "")).collect + + sql(s"""drop table if exists t3""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_054 + test("Batch_sort_Loading_001-01-01-01_001-TC_054", Include) { + sql(s"""drop table if exists t3""").collect + sql( + s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype String, + |serialname String, salary Int,floatField float) + | STORED BY 'carbondata'""".stripMargin.replaceAll(System.lineSeparator, "")).collect + + sql( + s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' + | into table t3 options('ALL_DICTIONARY_PATH'= + | '$resourcesPath/Data/columndict/data.dictionary','single_pass'='true')""".stripMargin + .replaceAll(System.lineSeparator, "")).collect + sql(s"""drop table if exists t3""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_056 + test("Batch_sort_Loading_001-01-01-01_001-TC_056", Include) { + sql(s"""drop table if exists uniqdata20a""").collect + sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='LOCAL_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='NO_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='NO_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='LOCAL_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + + sql(s"""select * from uniqdata20a""").collect + sql(s"""drop table if exists uniqdata20a""").collect + + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_057 + test("Batch_sort_Loading_001-01-01-01_001-TC_057", Include) { + sql(s"""drop table if exists uniqdata20a""").collect + sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/batchsort/1000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' ,'SORT_SCOPE'='BATCH_SORT','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""".stripMargin).collect + + sql(s"""alter table uniqdata20a compact 'minor'""").collect + sql(s"""drop table if exists uniqdata20a""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_058 + test("Batch_sort_Loading_001-01-01-01_001-TC_058", Include) { + sql(s"""drop table if exists uniqdata20a""").collect + sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + + intercept[Exception] { + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' , 'SORT_SCOPE'='ABCXYZ',âSINGLE_PASSâ=âtrueâ,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a_hive """).collect + } + sql(s"""drop table if exists uniqdata20a""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_059 + test("Batch_sort_Loading_001-01-01-01_001-TC_059", Include) { + sql(s"""drop table if exists uniqdata20a""").collect + intercept[Exception] { + sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' , 'SORT_SCOPE'='null',âSINGLE_PASSâ=âtrueâ,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a_hive """).collect + } + sql(s"""drop table if exists uniqdata20a""").collect + } + + //Batch_sort_Loading_001-01-01-01_001-TC_060 + test("Batch_sort_Loading_001-01-01-01_001-TC_060", Include) { + sql(s"""drop table if exists uniqdata20b""").collect + sql(s"""drop table if exists uniqdata20c""").collect + sql(s"""CREATE TABLE uniqdata20b (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20b OPTIONS('DELIMITER'=',','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""CREATE TABLE uniqdata20c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""insert into uniqdata20c select * from uniqdata20b""") + sql(s"""drop table if exists uniqdata20b""").collect + sql(s"""drop table if exists uniqdata20c""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_061 + test("Batch_sort_Loading_001-01-01-01_001-TC_061", Include) { + sql(s"""drop TABLE if exists uniqdata_h""").collect + sql(s"""drop TABLE if exists uniqdata_c""").collect + sql(s"""CREATE TABLE uniqdata_h (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""").collect + sql(s"""load data inpath '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata_h""").collect + sql(s"""CREATE TABLE uniqdata_c (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""insert into uniqdata_c select * from uniqdata_h""") + sql(s"""drop table if exists uniqdata_h""").collect + sql(s"""drop table if exists uniqdata_c""").collect + } + + + //Batch_sort_Loading_001-01-01-01_001-TC_064 +// test("Batch_sort_Loading_001-01-01-01_001-TC_064", Include) { +// sql(s"""drop table if exists uniqdata""").collect +// sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='20160302,20150302')""").collect +// sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect +// sql(s"""drop table if exists uniqdata""").collect +// } + + override def afterAll { + } +} \ No newline at end of file