[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r178216676
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
---
@@ -167,59 +170,60 @@ class TestTimeseriesDataLoad extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
}
- test("test Year level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_year"),
+
+ test("test timeseries table selection 1: year level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_year"),
Seq(Row(4)))
}
- test("test month level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_month"),
+ test("test timeseries table selection 2: month level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_month"),
Seq(Row(4)))
}
- test("test day level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_day"),
+ test("test timeseries table selection 3: day level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_day"),
--- End diff --
I confirmed with jacky and raghu
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user asfgit closed the pull request at: https://github.com/apache/carbondata/pull/1856 ---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174406656
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -17,18 +17,28 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import
org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException,
MalformedDataMapCommandException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import
org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.util.CarbonProperties
-class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
+class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll
with BeforeAndAfterEach{
val timeSeries = TIMESERIES.toString
+ var timestampFormat: String = _
--- End diff --
ok
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174406193
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala
---
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesUnsupportedSuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
--- End diff --
you can find these in preaggregate test cases. No need check the same for
timeseries because the base code is the same
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174404853
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
checkExistence(sql("DESC FORMATTED mainTable_agg1"), true,
"maintable_age_sum")
}
+ test("test timeseries create table 32: should support if not exists,
create when same table not exists") {
+sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+sql(
+ s"""
+ |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
+ |USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'YEAR_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_year")
+checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true,
"maintable_age_sum")
+ }
+
test("test timeseries create table 20: don't support 'create datamap if
exists'") {
val e: Exception = intercept[AnalysisException] {
sql(
s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
""".stripMargin)
}
assert(e.getMessage.contains("identifier matching regex"))
}
+ test("test timeseries create table 26: test different data type") {
+sql("drop table if exists dataTable")
+sql(
+ s"""
+ | CREATE TABLE dataTable(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | dataTime timestamp
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+
+sql(
+ s"""CREATE DATAMAP agg0_hour ON TABLE dataTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT
+ | dataTime,
+ | SUM(intField),
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+ | FROM dataTable
+ | GROUP BY
+ | dataTime,
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE dataTable"), true,
"datatable_agg0_hour")
+sql("DROP TABLE IF EXISTS dataTable")
+ }
+
+ test("test timeseries create table 27: test data map name") {
--- End diff --
but it has covered this scenario also right??
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174130738
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
---
@@ -33,16 +33,15 @@ class TestSortColumns extends QueryTest with
BeforeAndAfterAll {
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-")
SparkUtil4Test.createTaskMockUp(sqlContext)
-dropTable
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
"dd-MM-")
+dropTestTables
--- End diff --
I add common method QueryTest for all test case in
org.apache.spark.sql.test.util.QueryTest, so subclass should change the method
name
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174129743
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
---
@@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest
with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
}
- test("test PreAggregate table selection 1") {
-val df = sql("select mytime from mainTable group by mytime")
+ test("test timeseries table selection 1") {
+val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
- test("test PreAggregate table selection 2") {
-val df = sql("select timeseries(mytime,'hour') from mainTable group by
timeseries(mytime,'hour')")
+ test("test timeseries table selection 2") {
+val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY
TIMESERIES(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 3") {
-val df = sql("select timeseries(mytime,'milli') from mainTable group
by timeseries(mytime,'milli')")
-preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ test("test timeseries table selection 3: No enum constant MILLI") {
+val e = intercept[Exception] {
+ val df = sql(
+"""
+ | SELECT TIMESERIES(mytime,'milli')
+ | FROM mainTable
+ | GROUP BY TIMESERIES(mytime,'milli')
+""".stripMargin)
+ preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ df.show()
+}
+assert(e.getMessage.contains(
+ "No enum constant
org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
}
- test("test PreAggregate table selection 4") {
-val df = sql("select timeseries(mytime,'year') from mainTable group by
timeseries(mytime,'year')")
+ test("test timeseries table selection 4") {
+val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY
TIMESERIES(mytime,'year')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_year")
}
- test("test PreAggregate table selection 5") {
-val df = sql("select timeseries(mytime,'day') from mainTable group by
timeseries(mytime,'day')")
+ test("test timeseries table selection 5") {
+val df = sql("SELECT TIMESERIES(mytime,'day') FROM mainTable GROUP BY
TIMESERIES(mytime,'day')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_day")
}
- test("test PreAggregate table selection 6") {
-val df = sql("select timeseries(mytime,'month') from mainTable group
by timeseries(mytime,'month')")
+ test("test timeseries table selection 6") {
+val df = sql("SELECT TIMESERIES(mytime,'month') FROM mainTable GROUP
BY TIMESERIES(mytime,'month')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_month")
}
- test("test PreAggregate table selection 7") {
-val df = sql("select timeseries(mytime,'minute') from mainTable group
by timeseries(mytime,'minute')")
+ test("test timeseries table selection 7") {
+val df = sql("SELECT TIMESERIES(mytime,'minute') FROM mainTable GROUP
BY TIMESERIES(mytime,'minute')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_minute")
}
- test("test PreAggregate table selection 8") {
-val df = sql("select timeseries(mytime,'second') from mainTable group
by timeseries(mytime,'second')")
+ test("test timeseries table selection 8") {
+val df = sql("SELECT TIMESERIES(mytime,'second') FROM mainTable GROUP
BY TIMESERIES(mytime,'second')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_second")
}
- test("test PreAggregate table selection 9") {
-val df = sql("select timeseries(mytime,'hour') from mainTable where
timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour')")
-preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
+ test("test timeseries table selection 9") {
+val df = sql(
+ """
+| SELECT TIMESERIES(mytime,'hour')
+| FROM mainTable
+| WHERE TIMESERIES(mytime,'hour')='x'
+| GROUP BY TIMESERIES(mytime,'hour')
+ """.stripMargin)
+preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 10") {
-val df = sql("select timeseries(mytime,'hour') from mainTable where
timeseries(
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174128639
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
---
@@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest
with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
}
- test("test PreAggregate table selection 1") {
-val df = sql("select mytime from mainTable group by mytime")
+ test("test timeseries table selection 1") {
+val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
- test("test PreAggregate table selection 2") {
-val df = sql("select timeseries(mytime,'hour') from mainTable group by
timeseries(mytime,'hour')")
+ test("test timeseries table selection 2") {
+val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY
TIMESERIES(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 3") {
-val df = sql("select timeseries(mytime,'milli') from mainTable group
by timeseries(mytime,'milli')")
-preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ test("test timeseries table selection 3: No enum constant MILLI") {
+val e = intercept[Exception] {
+ val df = sql(
+"""
+ | SELECT TIMESERIES(mytime,'milli')
+ | FROM mainTable
+ | GROUP BY TIMESERIES(mytime,'milli')
+""".stripMargin)
+ preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ df.show()
+}
+assert(e.getMessage.contains(
+ "No enum constant
org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
}
- test("test PreAggregate table selection 4") {
-val df = sql("select timeseries(mytime,'year') from mainTable group by
timeseries(mytime,'year')")
+ test("test timeseries table selection 4") {
+val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY
TIMESERIES(mytime,'year')")
--- End diff --
The operation should be upper case and table name should be down case
according to SQL syntax, Hive syntax also like this.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174128505
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala
---
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesUnsupportedSuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
--- End diff --
Can you tell me where is the test case of inser/load/Update/delete/alter
into timeseries aggregate table?
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174127244
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ var timestampFormat: String = _
--- End diff --
@sraghunandan suggested should not change the CarbonCommonConstants values
after running the test case, because other place maybe need this constants
value.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174127189
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
---
@@ -16,15 +16,28 @@
*/
package org.apache.carbondata.integration.spark.testsuite.timeseries
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-import
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import
org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException,
NoSuchDataMapException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.spark.exception.ProcessMetaDataException
class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll
with BeforeAndAfterEach {
+ val timeSeries = "timeseries"
+ var timestampFormat: String = _
--- End diff --
@sraghunandan suggested should not change the CarbonCommonConstants values
after running the test case, because other place maybe need this constants
value.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174126731
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
checkExistence(sql("DESC FORMATTED mainTable_agg1"), true,
"maintable_age_sum")
}
+ test("test timeseries create table 32: should support if not exists,
create when same table not exists") {
+sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+sql(
+ s"""
+ |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
+ |USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'YEAR_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_year")
+checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true,
"maintable_age_sum")
+ }
+
test("test timeseries create table 20: don't support 'create datamap if
exists'") {
val e: Exception = intercept[AnalysisException] {
sql(
s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
""".stripMargin)
}
assert(e.getMessage.contains("identifier matching regex"))
}
+ test("test timeseries create table 26: test different data type") {
+sql("drop table if exists dataTable")
+sql(
+ s"""
+ | CREATE TABLE dataTable(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | dataTime timestamp
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+
+sql(
+ s"""CREATE DATAMAP agg0_hour ON TABLE dataTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT
+ | dataTime,
+ | SUM(intField),
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+ | FROM dataTable
+ | GROUP BY
+ | dataTime,
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE dataTable"), true,
"datatable_agg0_hour")
+sql("DROP TABLE IF EXISTS dataTable")
+ }
+
+ test("test timeseries create table 27: test data map name") {
--- End diff --
test case 32 and test 27 is different, test case 32 is check carbon
should support if CREATE DATAMAP not exists
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174123968
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
checkExistence(sql("DESC FORMATTED mainTable_agg1"), true,
"maintable_age_sum")
}
+ test("test timeseries create table 32: should support if not exists,
create when same table not exists") {
+sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+sql(
+ s"""
+ |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
+ |USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'YEAR_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_year")
+checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true,
"maintable_age_sum")
+ }
+
test("test timeseries create table 20: don't support 'create datamap if
exists'") {
val e: Exception = intercept[AnalysisException] {
sql(
s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+ | USING '$timeSeries'
--- End diff --
ok
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174123530
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -17,18 +17,28 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import
org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException,
MalformedDataMapCommandException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import
org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.util.CarbonProperties
-class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
+class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll
with BeforeAndAfterEach{
val timeSeries = TIMESERIES.toString
+ var timestampFormat: String = _
--- End diff --
@sraghunandan suggested should not change the CarbonCommonConstants values
after running the test case, because other place maybe need this constants
value.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174122755
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
""".stripMargin)
}
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
test("test timeseries create table 1") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true,
"maintable_agg0_second")
-sql("drop datamap agg0_second on table mainTable")
+sql("DROP DATAMAP agg0_second ON TABLE mainTable")
--- End diff --
The operation should be upper case and table name should be down case
according to SQL syntax, Hive syntax also like this.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174119980
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -241,12 +311,12 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
assert(e.getMessage.equals("Only one granularity level can be
defined"))
}
- test("test timeseries create table 14: Only one granularity level can be
defined 2") {
-sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+ test("test timeseries create table 17: Only one granularity level can be
defined 2") {
+sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
--- End diff --
It's unreasonable to create pre-aggregate with second granularity, second
aggregate is used less than hour in reality life.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174119683
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
""".stripMargin)
}
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
test("test timeseries create table 1") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true,
"maintable_agg0_second")
-sql("drop datamap agg0_second on table mainTable")
+sql("DROP DATAMAP agg0_second ON TABLE mainTable")
}
test("test timeseries create table 2") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_hour"), true,
"maintable_agg0_hour")
-sql("drop datamap agg0_hour on table mainTable")
+sql("DROP DATAMAP agg0_hour ON TABLE mainTable")
}
+
test("test timeseries create table 3") {
checkExistence(sql("DESCRIBE FORMATTED maintable_agg0_day"), true,
"maintable_agg0_day")
-sql("drop datamap agg0_day on table mainTable")
+sql("DROP DATAMAP agg0_day ON TABLE mainTable")
}
+
test("test timeseries create table 4") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_month"), true,
"maintable_agg0_month")
-sql("drop datamap agg0_month on table mainTable")
+sql("DROP DATAMAP agg0_month ON TABLE mainTable")
}
+
test("test timeseries create table 5") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_year"), true,
"maintable_agg0_year")
-sql("drop datamap agg0_year on table mainTable")
+sql("DROP DATAMAP agg0_year ON TABLE mainTable")
}
- test("test timeseries create table 6") {
-intercept[Exception] {
+ test("test timeseries create table 6: TIMESERIES should define time
granularity") {
+sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+val e = intercept[MalformedCarbonCommandException] {
sql(
-s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SEC_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+s"""CREATE DATAMAP agg0_second ON TABLE mainTable USING
'$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'SEC_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
""".stripMargin)
}
+assert(e.getMessage.contains("TIMESERIES should define time
granularity"))
}
- test("test timeseries create table 7") {
-intercept[Exception] {
+ test("test timeseries create table 7: Granularity only support 1") {
+sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+val e = intercept[MalformedDataMapCommandException] {
sql(
s"""
| CREATE DATAMAP agg0_second ON TABLE mainTable
| USING '$timeSeries'
| DMPROPERTIES (
| 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='2')
+ | 'DAY_GRANULARITY'='1.5')
| AS SELECT dataTime, SUM(age) FROM mainTable
| GROUP BY dataTime
""".stripMargin)
}
+assert(e.getMessage.equals("Granularity only support 1"))
}
- test("test timeseries create table 8") {
-intercept[Exception] {
+ test("test timeseries create table 8: Granularity only support 1") {
+dropDataMaps("mainTable", "agg1_hour")
+val e = intercept[MalformedCarbonCommandException] {
sql(
-s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
+s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'HOUR_GRANULARITY'='hour=-2')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+}
+assert(e.getMessage.contains("Granularity only support ")
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174066548
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
---
@@ -33,16 +33,15 @@ class TestSortColumns extends QueryTest with
BeforeAndAfterAll {
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-")
SparkUtil4Test.createTaskMockUp(sqlContext)
-dropTable
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
"dd-MM-")
+dropTestTables
--- End diff --
not required
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174066445
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
---
@@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest
with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
}
- test("test PreAggregate table selection 1") {
-val df = sql("select mytime from mainTable group by mytime")
+ test("test timeseries table selection 1") {
+val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
- test("test PreAggregate table selection 2") {
-val df = sql("select timeseries(mytime,'hour') from mainTable group by
timeseries(mytime,'hour')")
+ test("test timeseries table selection 2") {
+val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY
TIMESERIES(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 3") {
-val df = sql("select timeseries(mytime,'milli') from mainTable group
by timeseries(mytime,'milli')")
-preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ test("test timeseries table selection 3: No enum constant MILLI") {
+val e = intercept[Exception] {
+ val df = sql(
+"""
+ | SELECT TIMESERIES(mytime,'milli')
+ | FROM mainTable
+ | GROUP BY TIMESERIES(mytime,'milli')
+""".stripMargin)
+ preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ df.show()
+}
+assert(e.getMessage.contains(
+ "No enum constant
org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
}
- test("test PreAggregate table selection 4") {
-val df = sql("select timeseries(mytime,'year') from mainTable group by
timeseries(mytime,'year')")
+ test("test timeseries table selection 4") {
+val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY
TIMESERIES(mytime,'year')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_year")
}
- test("test PreAggregate table selection 5") {
-val df = sql("select timeseries(mytime,'day') from mainTable group by
timeseries(mytime,'day')")
+ test("test timeseries table selection 5") {
+val df = sql("SELECT TIMESERIES(mytime,'day') FROM mainTable GROUP BY
TIMESERIES(mytime,'day')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_day")
}
- test("test PreAggregate table selection 6") {
-val df = sql("select timeseries(mytime,'month') from mainTable group
by timeseries(mytime,'month')")
+ test("test timeseries table selection 6") {
+val df = sql("SELECT TIMESERIES(mytime,'month') FROM mainTable GROUP
BY TIMESERIES(mytime,'month')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_month")
}
- test("test PreAggregate table selection 7") {
-val df = sql("select timeseries(mytime,'minute') from mainTable group
by timeseries(mytime,'minute')")
+ test("test timeseries table selection 7") {
+val df = sql("SELECT TIMESERIES(mytime,'minute') FROM mainTable GROUP
BY TIMESERIES(mytime,'minute')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_minute")
}
- test("test PreAggregate table selection 8") {
-val df = sql("select timeseries(mytime,'second') from mainTable group
by timeseries(mytime,'second')")
+ test("test timeseries table selection 8") {
+val df = sql("SELECT TIMESERIES(mytime,'second') FROM mainTable GROUP
BY TIMESERIES(mytime,'second')")
preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_second")
}
- test("test PreAggregate table selection 9") {
-val df = sql("select timeseries(mytime,'hour') from mainTable where
timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour')")
-preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
+ test("test timeseries table selection 9") {
+val df = sql(
+ """
+| SELECT TIMESERIES(mytime,'hour')
+| FROM mainTable
+| WHERE TIMESERIES(mytime,'hour')='x'
+| GROUP BY TIMESERIES(mytime,'hour')
+ """.stripMargin)
+preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 10") {
-val df = sql("select timeseries(mytime,'hour') from mainTable where
timeseries
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174064855
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
---
@@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest
with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
}
- test("test PreAggregate table selection 1") {
-val df = sql("select mytime from mainTable group by mytime")
+ test("test timeseries table selection 1") {
+val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
- test("test PreAggregate table selection 2") {
-val df = sql("select timeseries(mytime,'hour') from mainTable group by
timeseries(mytime,'hour')")
+ test("test timeseries table selection 2") {
+val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY
TIMESERIES(mytime,'hour')")
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
}
- test("test PreAggregate table selection 3") {
-val df = sql("select timeseries(mytime,'milli') from mainTable group
by timeseries(mytime,'milli')")
-preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ test("test timeseries table selection 3: No enum constant MILLI") {
+val e = intercept[Exception] {
+ val df = sql(
+"""
+ | SELECT TIMESERIES(mytime,'milli')
+ | FROM mainTable
+ | GROUP BY TIMESERIES(mytime,'milli')
+""".stripMargin)
+ preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ df.show()
+}
+assert(e.getMessage.contains(
+ "No enum constant
org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
}
- test("test PreAggregate table selection 4") {
-val df = sql("select timeseries(mytime,'year') from mainTable group by
timeseries(mytime,'year')")
+ test("test timeseries table selection 4") {
+val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY
TIMESERIES(mytime,'year')")
--- End diff --
why is this requried?
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174064540
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
---
@@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
-Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
-Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
-Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
-Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
-Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 14: load data into mainTable after
create timeseries datamap ON TABLE and SELECT sub table") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE TABLE main_table(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE main_table
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age)
+ | FROM main_table
+ | GROUP BY mytime""".stripMargin)
+
+
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+
+checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 15: load data into main_table
after create timeseries datamap ON TABLE 1") {
--- End diff --
this scenario is already covered in the previous tests
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174064693
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
---
@@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
-Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
-Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
-Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
-Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
-Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 14: load data into mainTable after
create timeseries datamap ON TABLE and SELECT sub table") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE TABLE main_table(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE main_table
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age)
+ | FROM main_table
+ | GROUP BY mytime""".stripMargin)
+
+
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+
+checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 15: load data into main_table
after create timeseries datamap ON TABLE 1") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE TABLE main_table(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE
main_table")
+
+sql(
+ s"""
+| CREATE DATAMAP agg0_minute ON TABLE main_table
+| USING '$timeSeries'
+| DMPROPERTIES (
+| 'event_time'='mytime',
+| 'MINUTE_GRANULARITY'='1')
+| AS SELECT mytime, SUM(age)
+| FROM main_table
+| GROUP BY mytime""".stripMargin)
+
+sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE
main_table")
+val df = sql(
+ """
+| SELECT
+| timeseries(mytime,'minute') AS minuteLevel,
+| SUM(age) AS SUM
+| FROM main_table
+| WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
+| GROUP BY
+| timeseries(mytime,'minute')
+| ORDER BY
+| timeseries(mytime,'minute')
+ """.stripMargin)
+
+checkAnswer(df,
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
+Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
+ }
+
+ test("test timeseries table selection 16: load data into main_table
after create timeseries datamap ON TABLE 2") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE T
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174064625
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
---
@@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
-Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
-Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
-Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
-Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
-Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 14: load data into mainTable after
create timeseries datamap ON TABLE and SELECT sub table") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE TABLE main_table(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE main_table
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age)
+ | FROM main_table
+ | GROUP BY mytime""".stripMargin)
+
+
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO
TABLE main_table")
+
+checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
+Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
+Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
+Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
+Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
+Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
+ }
+
+ test("test timeseries table selection 15: load data into main_table
after create timeseries datamap ON TABLE 1") {
+sql("DROP TABLE IF EXISTS main_table")
+sql(
+ """
+| CREATE TABLE main_table(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE
main_table")
+
+sql(
+ s"""
+| CREATE DATAMAP agg0_minute ON TABLE main_table
+| USING '$timeSeries'
+| DMPROPERTIES (
+| 'event_time'='mytime',
+| 'MINUTE_GRANULARITY'='1')
+| AS SELECT mytime, SUM(age)
+| FROM main_table
+| GROUP BY mytime""".stripMargin)
+
+sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE
main_table")
+val df = sql(
+ """
+| SELECT
+| timeseries(mytime,'minute') AS minuteLevel,
+| SUM(age) AS SUM
+| FROM main_table
+| WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
+| GROUP BY
+| timeseries(mytime,'minute')
+| ORDER BY
+| timeseries(mytime,'minute')
+ """.stripMargin)
+
+checkAnswer(df,
+ Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
+Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
+ }
+
+ test("test timeseries table selection 16: load data into main_table
after create timeseries datamap ON TABLE 2") {
--- End diff --
duplicate
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174064354
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
---
@@ -167,59 +170,60 @@ class TestTimeseriesDataLoad extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
}
- test("test Year level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_year"),
+
+ test("test timeseries table selection 1: year level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_year"),
Seq(Row(4)))
}
- test("test month level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_month"),
+ test("test timeseries table selection 2: month level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_month"),
Seq(Row(4)))
}
- test("test day level timeseries data validation1 ") {
-checkAnswer( sql("select count(*) from table_03_ag1_day"),
+ test("test timeseries table selection 3: day level timeseries data
validation1 ") {
+checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_day"),
--- End diff --
why is this required?
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174063930
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala
---
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesUnsupportedSuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
--- End diff --
I don't think this Unsupported suite is required. Already have test cases
for each scenario. No need to test the same for timeseries. It will increase
the build time.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174060918
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ var timestampFormat: String = _
--- End diff --
no need for this variable. simply set
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format
is after all.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174060602
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
---
@@ -16,15 +16,28 @@
*/
package org.apache.carbondata.integration.spark.testsuite.timeseries
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-import
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import
org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException,
NoSuchDataMapException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.spark.exception.ProcessMetaDataException
class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll
with BeforeAndAfterEach {
+ val timeSeries = "timeseries"
+ var timestampFormat: String = _
--- End diff --
no need for this variable. simply set
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format
is after all.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174060559
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -17,18 +17,28 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import
org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException,
MalformedDataMapCommandException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import
org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.util.CarbonProperties
-class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
+class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll
with BeforeAndAfterEach{
val timeSeries = TIMESERIES.toString
+ var timestampFormat: String = _
--- End diff --
no need for this variable. simply set
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format
is after all.
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174060207
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
checkExistence(sql("DESC FORMATTED mainTable_agg1"), true,
"maintable_age_sum")
}
+ test("test timeseries create table 32: should support if not exists,
create when same table not exists") {
+sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+sql(
+ s"""
+ |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
+ |USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'YEAR_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_year")
+checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true,
"maintable_age_sum")
+ }
+
test("test timeseries create table 20: don't support 'create datamap if
exists'") {
val e: Exception = intercept[AnalysisException] {
sql(
s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+ | USING '$timeSeries'
--- End diff --
revert this change. it is unncessary
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174059512
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
checkExistence(sql("DESC FORMATTED mainTable_agg1"), true,
"maintable_age_sum")
}
+ test("test timeseries create table 32: should support if not exists,
create when same table not exists") {
+sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+sql(
+ s"""
+ |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
+ |USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'YEAR_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_year")
+checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true,
"maintable_age_sum")
+ }
+
test("test timeseries create table 20: don't support 'create datamap if
exists'") {
val e: Exception = intercept[AnalysisException] {
sql(
s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
""".stripMargin)
}
assert(e.getMessage.contains("identifier matching regex"))
}
+ test("test timeseries create table 26: test different data type") {
+sql("drop table if exists dataTable")
+sql(
+ s"""
+ | CREATE TABLE dataTable(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | dataTime timestamp
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+
+sql(
+ s"""CREATE DATAMAP agg0_hour ON TABLE dataTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT
+ | dataTime,
+ | SUM(intField),
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+ | FROM dataTable
+ | GROUP BY
+ | dataTime,
+ | shortField,
+ | booleanField,
+ | intField,
+ | bigintField,
+ | doubleField,
+ | stringField,
+ | decimalField,
+ | charField,
+ | floatField
+""".stripMargin)
+checkExistence(sql("SHOW DATAMAP ON TABLE dataTable"), true,
"datatable_agg0_hour")
+sql("DROP TABLE IF EXISTS dataTable")
+ }
+
+ test("test timeseries create table 27: test data map name") {
--- End diff --
same as test timeseries create table 32. Please remove. avoid adding
duplicate test cases
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174058262
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
""".stripMargin)
}
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
test("test timeseries create table 1") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true,
"maintable_agg0_second")
-sql("drop datamap agg0_second on table mainTable")
+sql("DROP DATAMAP agg0_second ON TABLE mainTable")
}
test("test timeseries create table 2") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_hour"), true,
"maintable_agg0_hour")
-sql("drop datamap agg0_hour on table mainTable")
+sql("DROP DATAMAP agg0_hour ON TABLE mainTable")
}
+
test("test timeseries create table 3") {
checkExistence(sql("DESCRIBE FORMATTED maintable_agg0_day"), true,
"maintable_agg0_day")
-sql("drop datamap agg0_day on table mainTable")
+sql("DROP DATAMAP agg0_day ON TABLE mainTable")
}
+
test("test timeseries create table 4") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_month"), true,
"maintable_agg0_month")
-sql("drop datamap agg0_month on table mainTable")
+sql("DROP DATAMAP agg0_month ON TABLE mainTable")
}
+
test("test timeseries create table 5") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_year"), true,
"maintable_agg0_year")
-sql("drop datamap agg0_year on table mainTable")
+sql("DROP DATAMAP agg0_year ON TABLE mainTable")
}
- test("test timeseries create table 6") {
-intercept[Exception] {
+ test("test timeseries create table 6: TIMESERIES should define time
granularity") {
+sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+val e = intercept[MalformedCarbonCommandException] {
sql(
-s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SEC_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
+s"""CREATE DATAMAP agg0_second ON TABLE mainTable USING
'$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'SEC_GRANULARITY'='1')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
""".stripMargin)
}
+assert(e.getMessage.contains("TIMESERIES should define time
granularity"))
}
- test("test timeseries create table 7") {
-intercept[Exception] {
+ test("test timeseries create table 7: Granularity only support 1") {
+sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+val e = intercept[MalformedDataMapCommandException] {
sql(
s"""
| CREATE DATAMAP agg0_second ON TABLE mainTable
| USING '$timeSeries'
| DMPROPERTIES (
| 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='2')
+ | 'DAY_GRANULARITY'='1.5')
| AS SELECT dataTime, SUM(age) FROM mainTable
| GROUP BY dataTime
""".stripMargin)
}
+assert(e.getMessage.equals("Granularity only support 1"))
}
- test("test timeseries create table 8") {
-intercept[Exception] {
+ test("test timeseries create table 8: Granularity only support 1") {
+dropDataMaps("mainTable", "agg1_hour")
+val e = intercept[MalformedCarbonCommandException] {
sql(
-s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
+s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
+ |DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'HOUR_GRANULARITY'='hour=-2')
+ |AS SELECT dataTime, SUM(age) FROM mainTable
+ |GROUP BY dataTime
+""".stripMargin)
+}
+assert(e.getMessage.contains("Granularity only support "
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174058314
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -241,12 +311,12 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
assert(e.getMessage.equals("Only one granularity level can be
defined"))
}
- test("test timeseries create table 14: Only one granularity level can be
defined 2") {
-sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
+ test("test timeseries create table 17: Only one granularity level can be
defined 2") {
+sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
--- End diff --
why change just the name of the table??
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r174056762
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
---
@@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest
with BeforeAndAfterAll {
""".stripMargin)
}
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
test("test timeseries create table 1") {
checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true,
"maintable_agg0_second")
-sql("drop datamap agg0_second on table mainTable")
+sql("DROP DATAMAP agg0_second ON TABLE mainTable")
--- End diff --
why is this change necessary??
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r167147440
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
---
@@ -36,16 +48,168 @@ class TestTimeSeriesDropSuite extends QueryTest with
BeforeAndAfterAll with Befo
""".stripMargin)
}
- test("test timeseries drop datamap 1: drop datamap should throw
exception if no datamap") {
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
+ test("test timeseries drop datamap 1: drop datamap should throw
exception") {
// DROP DATAMAP DataMapName if the DataMapName not exists
checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
val e: Exception = intercept[Exception] {
sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
}
-assert(e.getMessage.equals("Datamap with name agg1_month does not
exist under table mainTable"))
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 2: should support drop datamap IF
EXISTS") {
+// DROP DATAMAP IF EXISTS DataMapName
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
+assert(true)
+ }
+
+ test("test timeseries drop datamap 3: should support drop datamap") {
+sql(
+ s"""
+ | CREATE DATAMAP agg1_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) from mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+
+// Before DROP DATAMAP
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_month")
+
+// DROP DATAMAP DataMapName
+sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+}
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 4: should support drop datamap with
IF EXISTS") {
+sql(
+ s"""
+ | CREATE DATAMAP agg1_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) from mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+// DROP DATAMAP IF EXISTS DataMapName
+sql(s"DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false,
"agg1_year")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_month")
+
+// DROP DATAMAP DataMapName
+sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+}
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 5: drop datamap when table not
exists") {
--- End diff --
ok,done
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user sraghunandan commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r167120975
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
---
@@ -36,16 +48,168 @@ class TestTimeSeriesDropSuite extends QueryTest with
BeforeAndAfterAll with Befo
""".stripMargin)
}
- test("test timeseries drop datamap 1: drop datamap should throw
exception if no datamap") {
+ override def afterEach(): Unit = {
+dropDataMaps("mainTable", "agg1_second", "agg1_minute",
+ "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
+ }
+
+ test("test timeseries drop datamap 1: drop datamap should throw
exception") {
// DROP DATAMAP DataMapName if the DataMapName not exists
checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
val e: Exception = intercept[Exception] {
sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
}
-assert(e.getMessage.equals("Datamap with name agg1_month does not
exist under table mainTable"))
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 2: should support drop datamap IF
EXISTS") {
+// DROP DATAMAP IF EXISTS DataMapName
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
+assert(true)
+ }
+
+ test("test timeseries drop datamap 3: should support drop datamap") {
+sql(
+ s"""
+ | CREATE DATAMAP agg1_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) from mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+
+// Before DROP DATAMAP
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_month")
+
+// DROP DATAMAP DataMapName
+sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+}
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 4: should support drop datamap with
IF EXISTS") {
+sql(
+ s"""
+ | CREATE DATAMAP agg1_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'event_time'='dataTime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) from mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+// DROP DATAMAP IF EXISTS DataMapName
+sql(s"DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false,
"agg1_year")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true,
"agg1_month")
+
+// DROP DATAMAP DataMapName
+sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false,
"agg1_month")
+val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+}
+assert(e.getMessage.contains(
+ "Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 5: drop datamap when table not
exists") {
--- End diff --
Modify the existing test case to catch MalformedCarbonCommandException
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r165051330
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+SparkUtil4Test.createTaskMockUp(sqlContext)
+CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ }
+
+ override protected def beforeEach(): Unit = {
+sql("drop table if exists mainTable")
+sql(
+ """
+| CREATE TABLE mainTable(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
+ }
+
+ val timeSeries = "preaggregate"
+
+ test("test timeseries match 1: select small size when there are >1
timeseries match") {
+
+dropDataMaps("maintable", "agg1_minute", "agg2_minute")
+sql(
+ s"""
+ | create datamap agg1 on table mainTable
--- End diff --
ok,done
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r165051297
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+SparkUtil4Test.createTaskMockUp(sqlContext)
+CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ }
+
+ override protected def beforeEach(): Unit = {
+sql("drop table if exists mainTable")
+sql(
+ """
+| CREATE TABLE mainTable(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
+ }
+
+ val timeSeries = "preaggregate"
+
+ test("test timeseries match 1: select small size when there are >1
timeseries match") {
+
+dropDataMaps("maintable", "agg1_minute", "agg2_minute")
+sql(
+ s"""
+ | create datamap agg1 on table mainTable
+ | using '$timeSeries'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='mytime',
+ | 'timeseries.hierarchy'='minute=1')
+ | as select mytime, sum(age), count(age), max(age), min(age)
+ | from mainTable
+ | group by mytime
+ """.stripMargin)
+
+val df1 = sql(
+ """
+| select
+| timeseries(mytime,'minute') as minuteLevel,
+| sum(age) as sum
+| from mainTable
+| where
+| timeseries(mytime,'minute')<'2016-02-23 01:02:00' and
+| timeseries(mytime,'minute')>='2016-02-23 01:01:00'
+| group by
+| timeseries(mytime,'minute')
+| order by
+| timeseries(mytime,'minute')
+ """.stripMargin)
+
+checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 01:01:00"),
60)))
+
+preAggTableValidator(df1.queryExecution.analyzed,
"maintable_agg1_minute")
+
+sql(
+ s"""
+ | create datamap agg2 on table mainTable
+ | using '$timeSeries'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='mytime',
+ | 'timeseries.hierarchy'='minute=1')
+ | as select mytime, sum(age)
+ | from mainTable
+ | group by mytime
+ """.stripMargin)
+
+val df2 = sql(
+ """
+| select
+| timeseries(mytime,'minute') as minuteLevel,
+| sum(age) as sum
+| from mainTable
+| where
+| timeseries(mytime,'minute')<'2016-02-23 01:02:00' and
+| timeseries(mytime,'minute')>='2016-02-23 01:01:00'
+| group by
+| timeseries(mytime,'minute')
+| order by
+| timeseries(mytime,'minute')
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r164660048
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+SparkUtil4Test.createTaskMockUp(sqlContext)
+CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ }
+
+ override protected def beforeEach(): Unit = {
+sql("drop table if exists mainTable")
+sql(
+ """
+| CREATE TABLE mainTable(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
+ }
+
+ val timeSeries = "preaggregate"
+
+ test("test timeseries match 1: select small size when there are >1
timeseries match") {
+
+dropDataMaps("maintable", "agg1_minute", "agg2_minute")
+sql(
+ s"""
+ | create datamap agg1 on table mainTable
--- End diff --
change datamap name to big_agg
---
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1856#discussion_r164659399
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
---
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestTimeSeriesMatchStrategySuite extends QueryTest with
BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+SparkUtil4Test.createTaskMockUp(sqlContext)
+CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ }
+
+ override protected def beforeEach(): Unit = {
+sql("drop table if exists mainTable")
+sql(
+ """
+| CREATE TABLE mainTable(
+| mytime timestamp,
+| name string,
+| age int)
+| STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into
table mainTable")
+ }
+
+ val timeSeries = "preaggregate"
+
+ test("test timeseries match 1: select small size when there are >1
timeseries match") {
+
+dropDataMaps("maintable", "agg1_minute", "agg2_minute")
+sql(
+ s"""
+ | create datamap agg1 on table mainTable
+ | using '$timeSeries'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='mytime',
+ | 'timeseries.hierarchy'='minute=1')
+ | as select mytime, sum(age), count(age), max(age), min(age)
+ | from mainTable
+ | group by mytime
+ """.stripMargin)
+
+val df1 = sql(
+ """
+| select
+| timeseries(mytime,'minute') as minuteLevel,
+| sum(age) as sum
+| from mainTable
+| where
+| timeseries(mytime,'minute')<'2016-02-23 01:02:00' and
+| timeseries(mytime,'minute')>='2016-02-23 01:01:00'
+| group by
+| timeseries(mytime,'minute')
+| order by
+| timeseries(mytime,'minute')
+ """.stripMargin)
+
+checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 01:01:00"),
60)))
+
+preAggTableValidator(df1.queryExecution.analyzed,
"maintable_agg1_minute")
+
+sql(
+ s"""
+ | create datamap agg2 on table mainTable
+ | using '$timeSeries'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='mytime',
+ | 'timeseries.hierarchy'='minute=1')
+ | as select mytime, sum(age)
+ | from mainTable
+ | group by mytime
+ """.stripMargin)
+
+val df2 = sql(
+ """
+| select
+| timeseries(mytime,'minute') as minuteLevel,
+| sum(age) as sum
+| from mainTable
+| where
+| timeseries(mytime,'minute')<'2016-02-23 01:02:00' and
+| timeseries(mytime,'minute')>='2016-02-23 01:01:00'
+| group by
+| timeseries(mytime,'minute')
+| order by
+| timeseries(mytime,'minute')
[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...
GitHub user xubo245 opened a pull request: https://github.com/apache/carbondata/pull/1856 [CARBONDATA-2073][CARBONDATA-1516][Tests] Add test cases for timeseries pre-aggregate table Be sure to do all of the following checklist to help us incorporate your contribution quickly and easily: - [ ] Any interfaces changed? No - [ ] Any backward compatibility impacted? No - [ ] Document update required? No - [ ] Testing done Add test cases for timeseries pre-aggregate table in package org.apache.carbondata.integration.spark.testsuite.timeseries - [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA. refer [CARBONDATA-1516] You can merge this pull request into a Git repository by running: $ git pull https://github.com/xubo245/carbondata PreAggregateTestForTimeSeries Alternatively you can review and apply these changes as the patch at: https://github.com/apache/carbondata/pull/1856.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #1856 commit 66d7d0d97a706032459f2dd6b01c95053f437f79 Author: xubo245 <601450868@...> Date: 2018-01-24T09:35:24Z [CARBONDATA-2073][CARBONDATA-1516][Tests] Add test cases for timeseries pre-aggregate table ---
