http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala deleted file mode 100644 index cdfe2e6..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.spark.testsuite.deleteTable - -import java.io.File - -import org.apache.carbondata.spark.exception.MalformedCarbonCommandException -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -/** - * test class for testing the create cube DDL. - */ -class TestDeleteTableNewDDL extends QueryTest with BeforeAndAfterAll { - - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val resource = currentDirectory + "/src/test/resources/" - - override def beforeAll: Unit = { - - sql("CREATE TABLE IF NOT EXISTS table1(empno Int, empname Array<String>, designation String, doj Timestamp, " - + "workgroupcategory Int, workgroupcategoryname String, deptno Int, deptname String, projectcode Int, " - + "projectjoindate Timestamp, projectenddate Timestamp , attendance Int,utilization Int,salary Int )" - + " STORED BY 'org.apache.carbondata.format' ") - sql("CREATE TABLE IF NOT EXISTS table2(empno Int, empname Array<String>, designation String, doj Timestamp, " - + "workgroupcategory Int, workgroupcategoryname String, deptno Int, deptname String, projectcode Int, " - + "projectjoindate Timestamp, projectenddate Timestamp , attendance Int,utilization Int,salary Int )" - + " STORED BY 'org.apache.carbondata.format' ") - - } - - // normal deletion case - test("drop table Test with new DDL") { - sql("drop table table1") - } - - test("test drop database cascade command") { - sql("create database testdb") - sql("use testdb") - sql("CREATE TABLE IF NOT EXISTS testtable(empno Int, empname string, utilization Int,salary Int)" - + " STORED BY 'org.apache.carbondata.format' ") - try { - sql("drop database testdb") - assert(false) - } catch { - case e : Exception => - } - sql("drop database testdb cascade") - try { - sql("use testdb") - assert(false) - } catch { - case e : Exception => - } - sql("use default") - } - - // deletion case with if exists - test("drop table if exists Test with new DDL") { - sql("drop table if exists table2") - - } - - // try to delete after deletion with if exists - test("drop table after deletion with if exists with new DDL") { - sql("drop table if exists table2") - - } - - // try to delete after deletion with out if exists. this should fail - test("drop table after deletion with new DDL") { - try { - sql("drop table table2") - fail("failed") // this should not be executed as exception is expected - } - catch { - case e: Exception => // pass the test case as this is expected - } - - - } - - test("drop table using case insensitive table name") { - // create table - sql( - "CREATE table CaseInsensitiveTable (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'" + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='ID, salary')" - ) - // table should drop wihout any error - sql("drop table caseInsensitiveTable") - - // Now create same table, it should not give any error. - sql( - "CREATE table CaseInsensitiveTable (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'" + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='ID, salary')" - ) - - } - - test("drop table using dbName and table name") { - // create table - sql( - "CREATE table default.table3 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'" + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='ID, salary')" - ) - // table should drop without any error - sql("drop table default.table3") - } - - - test("drop table and create table with different data type") { - sql( - "CREATE table dropTableTest1 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format' " - - ) - - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE dropTableTest1 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from dropTableTest1") - sql("drop table dropTableTest1") - - sql( - "CREATE table dropTableTest1 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary String) stored by 'org.apache.carbondata.format' " - ) - - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE dropTableTest1 " + - "OPTIONS('DELIMITER' = ',')") - - sql("select * from dropTableTest1") - - } - - - test("drop table and create table with dictionary exclude integer scenario") { - sql( - "CREATE table dropTableTest2 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='salary')" - ) - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE dropTableTest2 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from dropTableTest2") - sql("drop table dropTableTest2") - sql( - "CREATE table dropTableTest2 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary decimal) stored by 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='date')" - ) - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE dropTableTest2 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from dropTableTest2") - - } - - test("drop table and create table with dictionary exclude string scenario") { - sql("create database if not exists test") - sql( - "CREATE table test.dropTableTest3 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_INCLUDE'='salary')" - ) - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE test.dropTableTest3 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from test.dropTableTest3") - sql("drop table test.dropTableTest3") - sql( - "CREATE table test.dropTableTest3 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary decimal) stored by 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_EXCLUDE'='date')" - ) - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE test.dropTableTest3 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from test.dropTableTest3") - - } - - test("drop table and create table with same name but different cols") { - - sql( - "CREATE TABLE dropTableTest4 (imei string,age int,task bigint,name string,country string," + - "city string,sale int,num double,level decimal(10,3),quest bigint,productdate timestamp," + - "enddate timestamp,PointId double,score decimal(10,3))STORED BY 'org.apache.carbondata" + - ".format'") - sql( - "LOAD DATA INPATH './src/test/resources/big_int_Decimal.csv' INTO TABLE dropTableTest4 " + - "options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'COMPLEX_DELIMITER_LEVEL_1'='$'," + - "'COMPLEX_DELIMITER_LEVEL_2'=':', 'FILEHEADER'= '')") - sql("select * from dropTableTest4") - sql("drop table dropTableTest4") - sql( - "CREATE table dropTableTest4 (ID int, date String, country String, name " + - "String," + - "phonetype String, serialname String, salary decimal) stored by 'org.apache.carbondata" + - ".format' " + - "TBLPROPERTIES('DICTIONARY_EXCLUDE'='date')" - ) - sql( - "LOAD DATA LOCAL INPATH '" + resource + "dataretention1.csv' INTO TABLE dropTableTest4 " + - "OPTIONS('DELIMITER' = ',')") - sql("select * from dropTableTest4") - - - } - - - override def afterAll: Unit = { - - sql("drop table CaseInsensitiveTable") - sql("drop table dropTableTest1") - sql("drop table dropTableTest2") - sql("drop table test.dropTableTest3") - sql("drop database test") - sql("drop table dropTableTest4") - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala deleted file mode 100644 index 1f94646..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.spark.testsuite.describeTable - -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.Row - -import org.scalatest.BeforeAndAfterAll - -/** - * test class for describe table . - */ -class TestDescribeTable extends QueryTest with BeforeAndAfterAll { - - override def beforeAll: Unit = { - sql("DROP TABLE IF EXISTS Desc1") - sql("DROP TABLE IF EXISTS Desc2") - sql("CREATE TABLE Desc1(Dec1Col1 String, Dec1Col2 String, Dec1Col3 int, Dec1Col4 double) stored by 'carbondata'") - sql("DESC Desc1") - sql("DROP TABLE Desc1") - sql("CREATE TABLE Desc1(Dec2Col1 BigInt, Dec2Col2 String, Dec2Col3 Bigint, Dec2Col4 Decimal) stored by 'carbondata'") - sql("CREATE TABLE Desc2(Dec2Col1 BigInt, Dec2Col2 String, Dec2Col3 Bigint, Dec2Col4 Decimal) stored by 'carbondata'") - } - - test("test describe table") { - checkAnswer(sql("DESC Desc1"), sql("DESC Desc2")) - } - - test("test describe formatted table") { - checkExistence(sql("DESC FORMATTED Desc1"), true, - "Table Block Size : 1024 MB") - } - - override def afterAll: Unit = { - sql("DROP TABLE Desc1") - sql("DROP TABLE Desc2") - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/AllDataTypesTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/AllDataTypesTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/AllDataTypesTestCase.scala deleted file mode 100644 index edf353f..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/AllDataTypesTestCase.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -/** - * Test Class for detailed query on multiple datatypes - * @author N00902756 - * - */ - -class AllDataTypesTestCase extends QueryTest with BeforeAndAfterAll { - - override def beforeAll { - sql("CREATE TABLE alldatatypestable (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'") - sql("LOAD DATA LOCAL INPATH './src/test/resources/data.csv' INTO TABLE alldatatypestable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"); - - sql("CREATE TABLE alldatatypestable_hive (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int)row format delimited fields terminated by ','") - sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO TABLE alldatatypestable_hive"); - - } - - test("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestable where empname in ('arvind','ayushi') group by empno,empname,utilization") { - checkAnswer( - sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestable where empname in ('arvind','ayushi') group by empno,empname,utilization"), - sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestable_hive where empname in ('arvind','ayushi') group by empno,empname,utilization")) - } - - override def afterAll { - sql("drop table alldatatypestable") - sql("drop table alldatatypestable_hive") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala deleted file mode 100644 index b86886e..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -/** - * Test Class for aggregate query on multiple datatypes - * - */ -class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll { - - override def beforeAll { - sql("create table colgrp (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")") - sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')"); - sql("create table normal (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'") - sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table normal options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')"); - //column group with dictionary exclude before column group - sql("create table colgrp_dictexclude_before (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column1',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")") - sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_dictexclude_before options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')"); - //column group with dictionary exclude after column group - sql("create table colgrp_dictexclude_after (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column10',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")") - sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_dictexclude_after options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')"); - - } - - test("select all dimension query") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp"), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal")) - } - - test("select all dimension query with filter on columnar") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column1='column1666'"), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column1='column1666'")) - } - - test("select all dimension query with filter on column group dimension") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311'"), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311'")) - } - - test("select all dimension query with filter on two dimension from different column group") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column7='column74' "), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column7='column74'")) - } - - test("select all dimension query with filter on two dimension from same column group") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column4='column42' "), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column4='column42'")) - } - - test("select all dimension query with filter on two dimension one from column group other from columnar") { - checkAnswer( - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column5='column516' "), - sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column5='column516'")) - } - - test("select few dimension") { - checkAnswer( - sql("select column1,column3,column4,column5,column6,column9,column10 from colgrp"), - sql("select column1,column3,column4,column5,column6,column9,column10 from normal")) - } - - test("select count on column group") { - checkAnswer( - sql("select count(column2) from colgrp"), - sql("select count(column2) from normal")) - } - test("##ColumnGroup_DictionaryExcludeBefore select all dimension on column group and dictionary exclude table") { - checkAnswer( - sql("select * from colgrp_dictexclude_before"), - sql("select * from normal")) - } - test("##ColumnGroup_DictionaryExcludeBefore select all dimension query with filter on two dimension from same column group") { - checkAnswer( - sql("select * from colgrp_dictexclude_before where column3='column311' and column4='column42' "), - sql("select * from normal where column3='column311' and column4='column42'")) - } - test("##ColumnGroup_DictionaryExcludeAfter select all dimension on column group and dictionary exclude table") { - checkAnswer( - sql("select * from colgrp_dictexclude_after"), - sql("select * from normal")) - } - test("##ColumnGroup_DictionaryExcludeAfter select all dimension query with filter on two dimension from same column group") { - checkAnswer( - sql("select * from colgrp_dictexclude_after where column3='column311' and column4='column42' "), - sql("select * from normal where column3='column311' and column4='column42'")) - } - test("ExcludeFilter") { - checkAnswer( - sql("select * from colgrp where column3 != 'column311'"), - sql("select * from normal where column3 != 'column311'")) - - checkAnswer( - sql("select * from colgrp where column3 like 'column31%'"), - sql("select * from normal where column3 like 'column31%'")) - checkAnswer( - sql("select * from colgrp where column3 not like 'column31%'"), - sql("select * from normal where column3 not like 'column31%'")) - } - test("RowFilter") { - checkAnswer( - sql("select * from colgrp where column3 != column4"), - sql("select * from normal where column3 != column4")) - } - - test("Column Group not in order with schema") { - //Add column group in order different then schema - try { - sql("create table colgrp_disorder (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column7,column8),(column2,column3,column4)\")") - sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_disorder options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')"); - assert(true) - } catch { - case ex: Exception => assert(false) - } - - } - override def afterAll { - sql("drop table colgrp") - sql("drop table normal") - sql("drop table colgrp_dictexclude_before") - sql("drop table colgrp_dictexclude_after") - sql("drop table if exists colgrp_disorder") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala deleted file mode 100644 index 7e9c17f..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll -import org.apache.spark.sql.Row - -class ColumnPropertyValidationTestCase extends QueryTest with BeforeAndAfterAll { - override def beforeAll { - sql("""drop table if exists employee""") - } - - test("Validate ColumnProperties_ valid key") { - try { - sql("create table employee(empname String,empid String,city String,country String,gender String,salary Double) stored by 'org.apache.carbondata.format' tblproperties('columnproperties.gender.key'='value')") - assert(true) - sql("drop table employee") - } catch { - case e: Throwable =>assert(false) - } - } - test("Validate Dictionary include _ invalid key") { - try { - sql("create table employee(empname String,empid String,city String,country String,gender String,salary Double) stored by 'org.apache.carbondata.format' tblproperties('columnproperties.invalid.key'='value')") - assert(false) - sql("drop table employee") - } catch { - case e: Throwable =>assert(true) - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala deleted file mode 100644 index eeef1d9..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll -import org.apache.spark.sql.Row -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.util.CarbonProperties - -/** - * Test Class for verifying NO_DICTIONARY_COLUMN feature. - * - * @author S71955 - * - */ -class NO_DICTIONARY_COL_TestCase extends QueryTest with BeforeAndAfterAll { - - override def beforeAll { - //For the Hive table creation and data loading - sql("drop table if exists filtertestTable") - sql("drop table if exists NO_DICTIONARY_HIVE_6") - sql("drop table if exists NO_DICTIONARY_CARBON_6") - sql("drop table if exists NO_DICTIONARY_CARBON_7") - - sql( - "create table NO_DICTIONARY_HIVE_6(empno string,empname string,designation string,doj " + - "Timestamp,workgroupcategory int, " + - "workgroupcategoryname string,deptno int, deptname string, projectcode int, " + - "projectjoindate Timestamp,projectenddate Timestamp,attendance int, " - + "utilization int,salary int) row format delimited fields terminated by ',' " + - "tblproperties(\"skip.header.line.count\"=\"1\") " + - "" - ) - sql( - "load data local inpath './src/test/resources/datawithoutheader.csv' into table " + - "NO_DICTIONARY_HIVE_6" - ); - //For Carbon cube creation. - sql("CREATE TABLE NO_DICTIONARY_CARBON_6 (empno string, " + - "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " + - "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " + - "projectenddate Timestamp, designation String,attendance Int,utilization " + - "Int,salary Int) STORED BY 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empname,designation')" - ) - sql( - "LOAD DATA LOCAL INPATH './src/test/resources/data.csv' INTO TABLE NO_DICTIONARY_CARBON_6 " + - "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')" - ) - - sql("CREATE TABLE NO_DICTIONARY_CARBON_7 (empno string, " + - "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " + - "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " + - "projectenddate Timestamp, designation String,attendance Int,utilization " + - "Int,salary Int) STORED BY 'org.apache.carbondata.format' " + - "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno,empname,designation')" - ) - sql( - "LOAD DATA LOCAL INPATH './src/test/resources/data.csv' INTO TABLE NO_DICTIONARY_CARBON_7 " + - "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')" - ) - sql("CREATE TABLE filtertestTable (ID string,date Timestamp, country String, " + - "name String, phonetype String, serialname String, salary Int) " + - "STORED BY 'org.apache.carbondata.format' " + "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ID')" - ) - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") - sql( - s"LOAD DATA LOCAL INPATH './src/test/resources/data2.csv' INTO TABLE filtertestTable OPTIONS"+ - s"('DELIMITER'= ',', " + - s"'FILEHEADER'= '')" - ); - - } - - test("Count (*) with filter") { - checkAnswer( - sql("select count(*) from NO_DICTIONARY_CARBON_6 where empno='11'"), - Seq(Row(1)) - ) - } - - test("Detail Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_6"), - Seq(Row("11"), Row("12"), Row("13"), Row("14"), Row("15"), Row("16"), Row("17"), Row("18"), Row("19"), Row("20")) - ) - - - } - - test("Detail Query with NO_DICTIONARY_COLUMN with Like range filter") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_7 where empno like '12%'"), - Seq(Row("12")) - ) - } - - test("Detail Query with NO_DICTIONARY_COLUMN with greater than range filter") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_7 where empno>'19'"), - Seq(Row("20")) - ) - } - - test("Detail Query with NO_DICTIONARY_COLUMN with in filter Compare With HIVE RESULT") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_6 where empno in('11','12','13')"), - Seq(Row("11"), Row("12"), Row("13")) - ) - } - test("Detail Query with NO_DICTIONARY_COLUMN with not in filter Compare With HIVE RESULT") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_6 where empno not in('11','12','13','14','15','16','17')"), - Seq(Row("18"), Row("19"), Row("20")) - ) - } - - test("Detail Query with NO_DICTIONARY_COLUMN with equals filter Compare With HIVE RESULT") { - - - checkAnswer( - sql("select empno from NO_DICTIONARY_CARBON_6 where empno='17'"), - Seq(Row("17")) - ) - } - test("Detail Query with NO_DICTIONARY_COLUMN with IS NOT NULL filter") { - - - checkAnswer( - sql("select id from filtertestTable where id is not null"), - Seq(Row("4"),Row("6"),Row("abc")) - ) - } -test("filter with arithmetic expression") { - checkAnswer( - sql("select id from filtertestTable " + "where id+2 = 6"), - Seq(Row("4")) - ) - } - test("Detail Query with NO_DICTIONARY_COLUMN with equals multiple filter Compare With HIVE " + - "RESULT" - ) { - - - checkAnswer( - sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno='17'"), - sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno='17'") - ) - } - - - test("ORDER Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql("select empno from NO_DICTIONARY_HIVE_6 order by empno"), - sql("select empno from NO_DICTIONARY_CARBON_6 order by empno") - ) - } - //TODO need to add filter test cases for no dictionary columns - // - // test("Filter Query with NO_DICTIONARY_COLUMN and DICTIONARY_COLUMN Compare With HIVE - // RESULT") { - // - // checkAnswer( - // sql("select empno from NO_DICTIONARY_HIVE_6 where empno=15 and deptno=12"), - // sql("select empno from NO_DICTIONARY_CARBON_6 where empno=15 and deptno=12")) - // } - - test("Distinct Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql("select count(distinct empno) from NO_DICTIONARY_HIVE_6"), - sql("select count(distinct empno) from NO_DICTIONARY_CARBON_6") - ) - } - test("Sum Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql("select sum(empno) from NO_DICTIONARY_HIVE_6"), - sql("select sum(empno) from NO_DICTIONARY_CARBON_6") - ) - } - - test("average Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql("select avg(empno) from NO_DICTIONARY_HIVE_6"), - sql("select avg(empno) from NO_DICTIONARY_CARBON_6") - ) - } - - - test("Multiple column group Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql( - "select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 group by empno,empname," + - "workgroupcategory" - ), - sql( - "select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 group by empno," + - "empname,workgroupcategory" - ) - ) - } - - test("Multiple column Detail Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") { - - checkAnswer( - sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6"), - sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 ") - ) - } - - - override def afterAll { - sql("drop table if exists filtertestTables") - //sql("drop cube NO_DICTIONARY_CARBON_1") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/IntegerDataTypeTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/IntegerDataTypeTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/IntegerDataTypeTestCase.scala deleted file mode 100644 index db94fd9..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/IntegerDataTypeTestCase.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -/** - * Test Class for detailed query on Integer datatypes - * @author N00902756 - * - */ -class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll { - - override def beforeAll { - sql("CREATE TABLE integertypetable (empno int, workgroupcategory string, deptno int, projectcode int,attendance int) STORED BY 'org.apache.carbondata.format'") - sql("LOAD DATA LOCAL INPATH './src/test/resources/data.csv' INTO TABLE integertypetable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')") - } - - test("select empno from integertypetable") { - checkAnswer( - sql("select empno from integertypetable"), - Seq(Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row(17), Row(18), Row(19), Row(20))) - } - - override def afterAll { - sql("drop table integertypetable") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala deleted file mode 100644 index 9ed969a..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -/** - * Test Class for detailed query on Int and BigInt of No Dictionary col - * - */ -class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll { - - override def beforeAll { - sql("DROP TABLE IF EXISTS carbonTable") - sql("DROP TABLE IF EXISTS hiveTable") - sql("DROP TABLE IF EXISTS carbonEmpty") - sql("DROP TABLE IF EXISTS hiveEmpty") - sql("CREATE TABLE carbonTable (imei String, age Int, num BigInt) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='age,num')") - sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegtiveNumber.csv' INTO TABLE carbonTable") - sql("CREATE TABLE hiveTable (imei String, age Int, num BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','") - sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegeativewithoutHeader.csv' INTO TABLE hiveTable") - - sql("CREATE TABLE carbonEmpty (cust_id int, cust_name String, active_emui_version String, bob timestamp, bigint_column bigint) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='cust_name,active_emui_version')") - sql("LOAD DATA LOCAL INPATH './src/test/resources/dataWithEmptyRows.csv' INTO TABLE carbonEmpty OPTIONS('FILEHEADER'='cust_id,cust_name,active_emui_version,bob,bigint_column')") - sql("CREATE TABLE hiveEmpty (cust_id int, cust_name String, active_emui_version String, bob timestamp, bigint_column bigint) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','") - sql("LOAD DATA LOCAL INPATH './src/test/resources/dataWithEmptyRows.csv' INTO TABLE hiveEmpty") - } - - test("SELECT IntType FROM carbonTable") { - checkAnswer( - sql("SELECT imei,age FROM carbonTable ORDER BY imei"), - sql("SELECT imei,age FROM hiveTable ORDER BY imei") - ) - } - - test("SELECT BigIntType FROM carbonTable") { - checkAnswer( - sql("SELECT imei,num FROM carbonTable ORDER BY imei"), - sql("SELECT imei,num FROM hiveTable ORDER BY imei") - ) - } - - test("test load data with one row that all no dictionary column values are empty") { - checkAnswer( - sql("SELECT cust_name,active_emui_version FROM carbonEmpty"), - sql("SELECT cust_name,active_emui_version FROM hiveEmpty") - ) - } - - override def afterAll { - sql("DROP TABLE IF EXISTS carbonTable") - sql("DROP TABLE IF EXISTS hiveTable") - sql("DROP TABLE IF EXISTS carbonEmpty") - sql("DROP TABLE IF EXISTS hiveEmpty") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala deleted file mode 100644 index 029c355..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -import org.apache.carbondata.core.datastorage.store.impl.FileFactory -import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType - -class SubqueryWithFilterAndSortTestCase extends QueryTest with BeforeAndAfterAll { - val tempDirPath = "./src/test/resources/temp" - val tempFilePath = "./src/test/resources/temp/subqueryfilterwithsort.csv" - - override def beforeAll { - FileFactory.mkdirs(tempDirPath,FileType.LOCAL) - sql("drop table if exists subqueryfilterwithsort") - sql("drop table if exists subqueryfilterwithsort_hive") - sql("CREATE TABLE subqueryfilterwithsort (name String, id int) STORED BY 'org.apache.carbondata.format'") - sql("CREATE TABLE subqueryfilterwithsort_hive (name String, id int)row format delimited fields terminated by ','") - val data ="name_a,1\nname_b,2\nname_c,3\nname_d,4\nname_e,5\nname_f,6" - writedata(tempFilePath, data) - sql(s"LOAD data local inpath '${tempFilePath}' into table subqueryfilterwithsort options('fileheader'='name,id')") - sql(s"LOAD data local inpath '${tempFilePath}' into table subqueryfilterwithsort_hive") - } - - test("When the query has sub-query with sort and has '=' filter") { - try { - checkAnswer(sql("select name,id from (select * from subqueryfilterwithsort order by id)t where name='name_c' "), - sql("select name,id from (select * from subqueryfilterwithsort_hive order by id)t where name='name_c'")) - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } - } - - test("When the query has sub-query with sort and has 'like' filter") { - try { - checkAnswer(sql("select name,id from (select * from subqueryfilterwithsort order by id)t where name like 'name%' "), - sql("select name,id from (select * from subqueryfilterwithsort_hive order by id)t where name like 'name%'")) - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } - } - - def writedata(filePath: String, data: String) = { - val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath)) - dis.writeBytes(data.toString()) - dis.close() - } - def deleteFile(filePath: String) { - val file = FileFactory.getCarbonFile(filePath, FileFactory.getFileType(filePath)) - file.delete() - } - - override def afterAll { - sql("drop table if exists subqueryfilterwithsort") - sql("drop table if exists subqueryfilterwithsort_hive") - deleteFile(tempFilePath) - deleteFile(tempDirPath) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala deleted file mode 100644 index 93f96d8..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.spark.testsuite.detailquery - -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.scalatest.BeforeAndAfterAll - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.datastorage.store.impl.FileFactory -import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType -import org.apache.carbondata.core.util.CarbonProperties - -/** - * Created by on 8/8/2016. - */ -class ValueCompressionDataTypeTestCase extends QueryTest with BeforeAndAfterAll { - val tempDirPath = "./src/test/resources/temp" - - override def beforeAll { - FileFactory.mkdirs(tempDirPath,FileType.LOCAL) - } - - test("ActualDataType:double,ChangedDatatype:Short,CompressionType:NonDecimalMaxMin") { - val tempFilePath = "./src/test/resources/temp/double2short.csv" - try { - sql("CREATE TABLE double2short (name String, value double) STORED BY 'org.apache.carbondata.format'") - sql("CREATE TABLE double2short_hive (name String, value double)row format delimited fields terminated by ','") - val data ="a,3.141111\nb,3.141212\nc,3.141313\nd,3.141515\ne,3.141616\nf,3.141616\ng,3.141717\nh,3.141818"; - writedata(tempFilePath, data) - sql(s"LOAD data local inpath '${tempFilePath}' into table double2short options('fileheader'='name,value')") - sql(s"LOAD data local inpath '${tempFilePath}' into table double2short_hive") - checkAnswer(sql("select * from double2short"), - sql("select * from double2short_hive")) - - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } finally { - sql("drop table if exists double2short") - sql("drop table if exists double2short_hive") - deleteFile(tempFilePath) - } - } - - test("ActualDataType:double,ChangedDatatype:byte,CompressionType:NonDecimalMaxMin") { - val tempFilePath = "./src/test/resources/temp/double2byte.csv" - try { - sql("CREATE TABLE double2byte (name String, value double) STORED BY 'org.apache.carbondata.format'") - sql("CREATE TABLE double2byte_hive (name String, value double)row format delimited fields terminated by ','") - val data ="a,4.200001\nb,4.200009"; - writedata(tempFilePath, data) - sql(s"LOAD data local inpath '${tempFilePath}' into table double2byte options('fileheader'='name,value')") - sql(s"LOAD data local inpath '${tempFilePath}' into table double2byte_hive") - checkAnswer(sql("select * from double2byte"), - sql("select * from double2byte_hive")) - - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } finally { - sql("drop table if exists double2byte") - sql("drop table if exists double2byte_hive") - deleteFile(tempFilePath) - } - } - - test("When the values of Double datatype are negative values") { - val tempFilePath = "./src/test/resources/temp/doubleISnegtive.csv" - try { - sql("drop table if exists doubleISnegtive") - sql("drop table if exists doubleISnegtive_hive") - sql("CREATE TABLE doubleISnegtive (name String, value double) STORED BY 'org.apache.carbondata.format'") - sql("CREATE TABLE doubleISnegtive_hive (name String, value double)row format delimited fields terminated by ','") - val data ="a,-7489.7976000000\nb,-11234567489.797\nc,-11234567489.7\nd,-1.2\ne,-2\nf,-11234567489.7976000000\ng,-11234567489.7976000000" - writedata(tempFilePath, data) - sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive options('fileheader'='name,value')") - sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive_hive") - - checkAnswer(sql("select * from doubleISnegtive"), - sql("select * from doubleISnegtive_hive")) - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } finally { - sql("drop table if exists doubleISnegtive") - sql("drop table if exists doubleISnegtive_hive") - deleteFile(tempFilePath) - } - } - - test("When the values of Double datatype have both postive and negative values") { - val tempFilePath = "./src/test/resources/temp/doublePAN.csv" - try { - sql("drop table if exists doublePAN") - sql("drop table if exists doublePAN_hive") - sql("CREATE TABLE doublePAN (name String, value double) STORED BY 'org.apache.carbondata.format'") - sql("CREATE TABLE doublePAN_hive (name String, value double)row format delimited fields terminated by ','") - val data ="a,-7489.7976000000\nb,11234567489.797\nc,-11234567489.7\nd,-1.2\ne,2\nf,-11234567489.7976000000\ng,11234567489.7976000000" - writedata(tempFilePath, data) - sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN options('fileheader'='name,value')") - sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN_hive") - - checkAnswer(sql("select * from doublePAN"), - sql("select * from doublePAN_hive")) - } catch{ - case ex:Exception => ex.printStackTrace() - assert(false) - } finally { - sql("drop table if exists doublePAN") - sql("drop table if exists doublePAN_hive") - deleteFile(tempFilePath) - } - } - - def writedata(filePath: String, data: String) = { - val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath)) - dis.writeBytes(data.toString()) - dis.close() - } - def deleteFile(filePath: String) { - val file = FileFactory.getCarbonFile(filePath, FileFactory.getFileType(filePath)) - file.delete() - } - - override def afterAll { - deleteFile(tempDirPath) - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala deleted file mode 100644 index e5cf72d..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.Date - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext.{sql, _} -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.scalatest.BeforeAndAfterAll - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true") - sql("drop table if exists directDictionaryTable ") - sql("drop table if exists directDictionaryTable_hive ") - sql( - "CREATE TABLE if not exists directDictionaryTable (empno int,doj date, " + - "salary int) " + - "STORED BY 'org.apache.carbondata.format'" - ) - - sql( - "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj date, " + - "salary int) " + - "row format delimited fields terminated by ','" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasamplefordate.csv" - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" + - "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" ) - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive") - sql("select * from directDictionaryTable_hive").show(false) - } catch { - case x: Throwable => - x.printStackTrace() - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("test direct dictionary for not null condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj is not null"), - Seq(Row(Date.valueOf("2016-03-14")), - Row(Date.valueOf("2016-04-14")) - ) - ) - } - - test("test direct dictionary for getting all the values") { - checkAnswer( - sql("select doj from directDictionaryTable"), - Seq(Row(Date.valueOf("2016-03-14")), - Row(Date.valueOf("2016-04-14")), - Row(null) - ) - ) - } - - test("test direct dictionary for not equals condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj != '2016-04-14 00:00:00'"), - Seq(Row(Date.valueOf("2016-03-14")) - ) - ) - } - - test("test direct dictionary for null condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj is null"), - Seq(Row(null) - ) - ) - } - - test("select doj from directDictionaryTable with equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj = '2016-03-14 00:00:00'"), - Seq(Row(Date.valueOf("2016-03-14"))) - ) - - } - - test("select doj from directDictionaryTable with regexp_replace equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') = '2016/03/14'"), - Seq(Row(Date.valueOf("2016-03-14"))) - ) - } - - test("select doj from directDictionaryTable with regexp_replace NOT IN filter") { - checkAnswer( - sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14')"), - sql("select doj from directDictionaryTable_hive where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14')") - ) - } - - test("select doj from directDictionaryTable with greater than filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj > '2016-03-14 00:00:00'"), - Seq(Row(Date.valueOf("2016-04-14"))) - ) - } - - test("select count(doj) from directDictionaryTable") { - checkAnswer( - sql("select count(doj) from directDictionaryTable"), - Seq(Row(2)) - ) - } - - override def afterAll { - sql("drop table directDictionaryTable") - sql("drop table directDictionaryTable_hive") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala deleted file mode 100644 index db2461e..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.{Date, Timestamp} - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.scalatest.BeforeAndAfterAll - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true") - sql( - """ - CREATE TABLE IF NOT EXISTS directDictionaryTable - (empno String, doj Date, salary Int) - STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='empno')""" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv" - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" - + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"); - } catch { - case x: Throwable => - x.printStackTrace() - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("select doj from directDictionaryTable") { - checkAnswer( - sql("select doj from directDictionaryTable"), - Seq(Row(Date.valueOf("2016-03-14")), - Row(Date.valueOf("2016-04-14")), - Row(null) - ) - ) - } - - - test("select doj from directDictionaryTable with equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'"), - Seq(Row(Date.valueOf("2016-03-14"))) - ) - - } - - test("select doj from directDictionaryTable with greater than filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"), - Seq(Row(Date.valueOf("2016-04-14"))) - ) - - } - - - override def afterAll { - sql("drop table directDictionaryTable") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala deleted file mode 100644 index 82b6783..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.{Date, Timestamp} - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.scalatest.BeforeAndAfterAll - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class DateDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - sql( - """CREATE TABLE IF NOT EXISTS timestampTyeNullData - (ID Int, dateField date, country String, - name String, phonetype String, serialname String, salary Int) - STORED BY 'org.apache.carbondata.format'""" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv" - sql("LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE timestampTyeNullData").collect(); - - } catch { - case x: Throwable => - x.printStackTrace() - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null") { - checkAnswer( - sql("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null"), - Seq(Row(Date.valueOf("2015-07-23")) - ) - ) - } - test("SELECT * FROM timestampTyeNullData where dateField is null") { - checkAnswer( - sql("SELECT dateField FROM timestampTyeNullData where dateField is null"), - Seq(Row(null) - )) - } - - override def afterAll { - sql("drop table timestampTyeNullData") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala deleted file mode 100644 index f166025..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.Timestamp - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.scalatest.BeforeAndAfterAll - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants -import org.apache.carbondata.core.util.CarbonProperties - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00.0") - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY, - TimeStampGranularityConstants.TIME_GRAN_SEC.toString - ) - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true") - sql("drop table if exists directDictionaryTable") - sql("drop table if exists directDictionaryTable_hive") - sql( - "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, salary int) " + - "STORED BY 'org.apache.carbondata.format'" - ) - - sql( - "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp, salary int) " + - "row format delimited fields terminated by ','" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv" - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" + - "('DELIMITER'= ',', 'QUOTECHAR'= '\"')") - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive") - } catch { - case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("test direct dictionary for not null condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj is not null"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")), - Row(Timestamp.valueOf("2016-04-14 15:00:09.0")) - ) - ) - } - - test("test direct dictionary for getting all the values") { - checkAnswer( - sql("select doj from directDictionaryTable"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")), - Row(Timestamp.valueOf("2016-04-14 15:00:09.0")), - Row(null) - ) - ) - } - - test("test direct dictionary for not equals condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09")) - ) - ) - } - - test("test direct dictionary for null condition") { - checkAnswer( - sql("select doj from directDictionaryTable where doj is null"), - Seq(Row(null) - ) - ) - } - - test("select doj from directDictionaryTable with equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09"))) - ) - - } - - test("select doj from directDictionaryTable with regexp_replace equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') = '2016/03/14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09"))) - ) - } - - test("select doj from directDictionaryTable with regexp_replace NOT IN filter") { - checkAnswer( - sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')"), - sql("select doj from directDictionaryTable_hive where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')") - ) - } - - test("select doj from directDictionaryTable with greater than filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09"))) - ) - } - - test("select count(doj) from directDictionaryTable") { - checkAnswer( - sql("select count(doj) from directDictionaryTable"), - Seq(Row(2)) - ) - } - - override def afterAll { - sql("drop table directDictionaryTable") - sql("drop table directDictionaryTable_hive") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala deleted file mode 100644 index 991b1bf..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.Timestamp - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.scalatest.BeforeAndAfterAll - -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants -import org.apache.carbondata.core.util.CarbonProperties - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00.0") - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY, - TimeStampGranularityConstants.TIME_GRAN_SEC.toString - ) - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true") - sql( - """ - CREATE TABLE IF NOT EXISTS directDictionaryTable - (empno String, doj Timestamp, salary Int) - STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='empno')""" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv" - sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" - + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"); - } catch { - case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("select doj from directDictionaryTable") { - checkAnswer( - sql("select doj from directDictionaryTable"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")), - Row(Timestamp.valueOf("2016-04-14 15:00:09.0")), - Row(null) - ) - ) - } - - - test("select doj from directDictionaryTable with equals filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09"))) - ) - - } - - test("select doj from directDictionaryTable with greater than filter") { - checkAnswer( - sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"), - Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09"))) - ) - - } - - - override def afterAll { - sql("drop table directDictionaryTable") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala deleted file mode 100644 index 9b7359f..0000000 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.testsuite.directdictionary - -import java.io.File -import java.sql.Timestamp - -import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.CarbonHiveContext._ -import org.apache.spark.sql.common.util.QueryTest -import org.apache.spark.sql.hive.HiveContext -import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants -import org.apache.carbondata.core.util.CarbonProperties -import org.scalatest.BeforeAndAfterAll - - -/** - * Test Class for detailed query on timestamp datatypes - * - * - */ -class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { - var hiveContext: HiveContext = _ - - override def beforeAll { - try { - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00.0") - CarbonProperties.getInstance() - .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY, - TimeStampGranularityConstants.TIME_GRAN_SEC.toString - ) - sql( - """CREATE TABLE IF NOT EXISTS timestampTyeNullData - (ID Int, dateField Timestamp, country String, - name String, phonetype String, serialname String, salary Int) - STORED BY 'org.apache.carbondata.format'""" - ) - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") - val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv" - sql("LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE timestampTyeNullData").collect(); - - } catch { - case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - } - } - - test("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null") { - checkAnswer( - sql("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null"), - Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0")) - ) - ) - } - test("SELECT * FROM timestampTyeNullData where dateField is null") { - checkAnswer( - sql("SELECT dateField FROM timestampTyeNullData where dateField is null"), - Seq(Row(null) - )) - } - - override def afterAll { - sql("drop table timestampTyeNullData") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") - CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") - } - -} \ No newline at end of file
