[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long string columns
Add validate for long string columns 1. long string columns cannot be sort_columns 2. long string columns cannot be dictionary include 3. long string columns cannot be dictionary exclude 4. long string columns can only be string columns This closes #2380 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/218a8deb Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/218a8deb Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/218a8deb Branch: refs/heads/carbonstore Commit: 218a8deb614e0dc160ab8c7e38c71d80711eb1a7 Parents: 091a28b Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Wed Jun 20 18:55:04 2018 +0800 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Thu Jun 21 12:00:34 2018 +0530 ---------------------------------------------------------------------- .../VarcharDataTypesBasicTestCase.scala | 57 ++++++++++++++++++++ .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +++- 2 files changed, 67 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala index 419b306..9ea3f1f 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala @@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi sql(s"drop table if exists $longStringTable") } + test("long string columns cannot be dictionary include") { + val exceptionCaught = intercept[Exception] { + sql( + s""" + | CREATE TABLE if not exists $longStringTable( + | id INT, name STRING, description STRING, address STRING, note STRING + | ) STORED BY 'carbondata' + | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_include'='address') + |""". + stripMargin) + } + assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is unsupported for long string datatype column: address")) + } + + test("long string columns cannot be dictionay exclude") { + val exceptionCaught = intercept[Exception] { + sql( + s""" + | CREATE TABLE if not exists $longStringTable( + | id INT, name STRING, description STRING, address STRING, note STRING + | ) STORED BY 'carbondata' + | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_exclude'='address') + |""". + stripMargin) + } + assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is unsupported for long string datatype column: address")) + } + + test("long string columns cannot be sort_columns") { + val exceptionCaught = intercept[Exception] { + sql( + s""" + | CREATE TABLE if not exists $longStringTable( + | id INT, name STRING, description STRING, address STRING, note STRING + | ) STORED BY 'carbondata' + | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 'SORT_COLUMNS'='name, address') + |""". + stripMargin) + } + assert(exceptionCaught.getMessage.contains("sort_columns is unsupported for long string datatype column: name")) + } + + test("long string columns can only be string columns") { + val exceptionCaught = intercept[Exception] { + sql( + s""" + | CREATE TABLE if not exists $longStringTable( + | id INT, name STRING, description STRING, address STRING, note STRING + | ) STORED BY 'carbondata' + | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note') + |""". + stripMargin) + } + assert(exceptionCaught.getMessage.contains("long_string_columns: id")) + assert(exceptionCaught.getMessage.contains("its data type is not string")) + } + private def prepareTable(): Unit = { sql( s""" http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index 350fc36..7c3cf65 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -752,7 +752,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } if (varcharCols.exists(x => x.equalsIgnoreCase(column))) { throw new MalformedCarbonCommandException( - s"sort_columns is unsupported for long string datatype column $column") + s"sort_columns is unsupported for long string datatype column: $column") } } } @@ -791,6 +791,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase() + " data type column: " + dictExcludeCol throw new MalformedCarbonCommandException(errorMsg) + } else if (varcharCols.exists(x => x.equalsIgnoreCase(dictExcludeCol))) { + throw new MalformedCarbonCommandException( + "DICTIONARY_EXCLUDE is unsupported for long string datatype column: " + + dictExcludeCol) } } } @@ -805,6 +809,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { " does not exist in table. Please check create table statement." throw new MalformedCarbonCommandException(errormsg) } + if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) { + throw new MalformedCarbonCommandException( + "DICTIONARY_INCLUDE is unsupported for long string datatype column: " + + distIncludeCol.trim) + } } }