Github user praveenmeenakshi56 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2422#discussion_r200237575
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
---
@@ -0,0 +1,1183 @@
+package org.apache.carbondata.spark.testsuite.localdictionary
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
+class LocalDictionarySupportAlterTableTest extends QueryTest with
BeforeAndAfterAll{
+
+ override protected def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS LOCAL1")
+ }
+
+ test("test alter table add column") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ |
'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
+ """.stripMargin)
+ sql("alter table local1 add columns (alt string)
tblproperties('local_dictionary_include'='alt')")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("20000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city,alt"))
+ }
+ }
+
+ test("test alter table add column default configs for local dictionary")
{
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_threshold'='20000','no_inverted_index'='name')
+ """.stripMargin)
+ sql("alter table local1 add columns (alt string)")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("20000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) =>
assert(row.get(1).toString.contains("name,city,alt"))
+ }
+ }
+
+ test("test alter table add column where same column is in dictionary
include and local dictionary include") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ |
'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string)
tblproperties('local_dictionary_include'='alt','dictionary_include'='alt')")
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: alt
specified in Dictionary " +
+ "include. Local Dictionary will not be generated for Dictionary
include columns. " +
+ "Please check the DDL."))
+ }
+
+ test("test alter table add column where duplicate columns present in
local dictionary include") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ |
'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string)
tblproperties('local_dictionary_include'='alt,alt')")
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains
Duplicate Columns: alt. " +
+ "Please check the DDL."))
+ }
+
+ test("test alter table add column where duplicate columns present in
local dictionary include/exclude")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ |
'local_dictionary_threshold'='20000','local_dictionary_include'='city',
+ | 'no_inverted_index'='name')
+ """.stripMargin)
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string) tblproperties" +
+ "('local_dictionary_include'='abc')")
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc
does not exist in table. " +
+ "Please check the DDL."))
+ val exception2 = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string) tblproperties" +
+ "('local_dictionary_exclude'='abc')")
+ }
+ assert(exception2.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc
does not exist in table. " +
+ "Please check the DDL."))
+ }
+
+ test("test alter table add column for datatype validation")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_include'='city', 'no_inverted_index'='name')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string,abc int)
tblproperties" +
+ "('local_dictionary_include'='abc')")
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is
not a String/complex " +
+ "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary
string/complex datatype" +
+ " column.Please check the DDL."))
+ }
+
+ test("test alter table add column where duplicate columns are present in
local dictionary include and exclude")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_include'='city', 'no_inverted_index'='name')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string,abc string)
tblproperties" +
+
"('local_dictionary_include'='abc','local_dictionary_exclude'='alt,abc')")
+ }
+ assert(exception.getMessage
+ .contains(
+ "Column ambiguity as duplicate column(s):abc is present in
LOCAL_DICTIONARY_INCLUDE " +
+ "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not
allowed."))
+ }
+
+ test("test alter table add column unsupported table property")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_include'='city', 'no_inverted_index'='name')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string,abc string)
tblproperties" +
+ "('local_dictionary_enable'='abc')")
+ }
+ assert(exception.getMessage
+ .contains(
+ "Unsupported Table property in add column:
local_dictionary_enable"))
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ "alter table local1 add columns (alt string,abc string)
tblproperties" +
+ "('local_dictionary_threshold'='10000')")
+ }
+ assert(exception1.getMessage
+ .contains(
+ "Unsupported Table property in add column:
local_dictionary_threshold"))
+ }
+
+ test("test alter table add column when main table is disabled for local
dictionary")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='false',
+ | 'local_dictionary_include'='city', 'no_inverted_index'='name')
+ """.stripMargin)
+ sql(
+ "alter table local1 add columns (alt string,abc string)
tblproperties" +
+ "('local_dictionary_include'='abc')")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+
+ checkExistence(sql("DESC FORMATTED local1"), false,
+ "Local Dictionary Include")
+ }
+
+ test("test local dictionary threshold for boundary values") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_threshold'='300000')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_threshold'='500')
+ """.stripMargin)
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test alter table add column for local dictionary include and
exclude configs")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_include'='city', 'no_inverted_index'='name')
+ """.stripMargin)
+ sql(
+ "alter table local1 add columns (alt string,abc string)
tblproperties" +
+
"('local_dictionary_include'='abc','local_dictionary_exclude'='alt')")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city,abc"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("alt"))
+ }
+ }
+
+ test("test local dictionary foer varchar datatype columns") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_include'='city',
+ | 'LONG_STRING_COLUMNS'='city')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary describe formatted only with default
configs")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name,city"))
+ }
+ }
+
+ test("test local dictionary for invalid threshold")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_threshold'='300000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test alter set for local dictionary enable to disable")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_threshold'='300000')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_enable'='false')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ checkExistence(sql("DESC FORMATTED local1"), false,
+ "Local Dictionary Threshold")
+ }
+
+ test("test alter set for local dictionary _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_threshold'='30000')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("30000"))
+ }
+ }
+
+
+ test("test alter set for local dictionary _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name") &&
!row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter set for local dictionary _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test alter set for local dictionary _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='city')")
+
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter set for local dictionary _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name,city')")
+
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name,city"))
+ }
+ }
+
+ test("test alter set for local dictionary _006")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city')")
+
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter set for local dictionary _007")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='city, ')")
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does
not exist in table. " +
+ "Please check the DDL."))
+ }
+
+ test("test alter set for local dictionary _008")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ }
+ assert(exception1.getMessage.contains("Column ambiguity as duplicate
column(s):name is present in LOCAL_DICTIONARY_INCLUDE and
LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
+ }
+
+ test("test alter set for local dictionary _009")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='id')")
+ }
+
assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE
column: id is not a String/complex datatype column.
LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary
string/complex datatype column."))
+ }
+
+ test("test alter set for local dictionary _010")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name')")
+ }
+
assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE
column: name specified in Dictionary include. Local Dictionary will not be
generated for Dictionary include columns. Please check the DDL."))
+ }
+
+ test("test alter set for local dictionary _011")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter set for local dictionary _012")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city, ')")
+ }
+
assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE
column: does not exist in table. Please check the DDL."))
+ }
+
+ test("test alter set for local dictionary _013")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ }
+
assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE
column: name specified in Dictionary include. Local Dictionary will not be
generated for Dictionary include columns. Please check the DDL."))
+ }
+
+ test("test alter set for local dictionary _014")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ }
+ assert(exception1.getMessage.contains("Column ambiguity as duplicate
column(s):name is present in LOCAL_DICTIONARY_INCLUDE and
LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
+ }
+
+ test("test alter set for local dictionary _015")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_enable'='false')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test alter set for local dictionary _016")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test alter set for local dictionary _017")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
+ """.stripMargin)
+
+ val exception1 = intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ }
+ assert(exception1.getMessage.contains("Column ambiguity as duplicate
column(s):name is present in LOCAL_DICTIONARY_INCLUDE and
LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
+ }
+
+ test("test alter unset for local dictionary _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
+ """.stripMargin)
+
+ sql("alter table local1 unset
tblproperties('local_dictionary_exclude')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test alter unset for local dictionary _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
+ """.stripMargin)
+
+ sql("alter table local1 unset
tblproperties('local_dictionary_exclude')")
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter set for local dictionary disable to enable")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='300000')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ checkExistence(sql("DESC FORMATTED local1"), false,
+ "Local Dictionary Threshold")
+ sql("alter table local1 set
tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='30000')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("30000"))
+ }
+ }
+
+ test("test alter set for local dictionary include valid and invalid")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city','dictionary_include'='add')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='id')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='city,city')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='add')")
+ }
+ }
+
+ test("test alter set for local dictionary exclude valid and invalid")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='city','dictionary_include'='add')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='id')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city,city')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='add')")
+ }
+ }
+
+ test("test alter set same column for local dictionary exclude and
include")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='city')
+ """.stripMargin)
+ intercept[Exception] {
+ sql(
+ "alter table local1 set
tblproperties('local_dictionary_include'='name'," +
+ "'local_dictionary_exclude'='name')")
+ }
+ }
+
+ test("test alter set for valid and invalid complex type as
include/exclude") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st
struct<s_id:int,
+ | s_name:string,s_city:array<string>>, dcity array<string>)
+ | STORED BY 'org.apache.carbondata.format'
+ |
tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,dcity',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary
Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(
+ row.get(1).toString.contains("city,dcity") &&
!row.get(1).toString.contains("name"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='dcity')")
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='st')")
+ val descFormatted2 = sql("describe formatted local1").collect
+ descFormatted2.find(_.get(0).toString.contains("Local Dictionary
Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st"))
+ }
+ descFormatted2.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,dcity"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='st'," +
+ "'local_dictionary_include'='dcity')")
+ val descFormatted3 = sql("describe formatted local1").collect
+ descFormatted3.find(_.get(0).toString.contains("Local Dictionary
Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st"))
+ }
+ descFormatted3.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("dcity"))
+ }
+ }
+
+ test("test alter set for invalid complex type as include/exclude") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st
struct<s_id:int,
+ | s_name:int,s_city:array<int>>, dcity array<int>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='dcity')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='st')")
+ }
+ }
+
+ test("test alter unset for local dictionary disable")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enable"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_enable')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ }
+
+ test("test alter unset for local dictionary enable local dict include")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_include')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name,city"))
+ }
+ }
+
+ test("test alter unset for local dictionary enable local dict exclude")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city',
+ | 'local_dictionary_exclude'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_include')")
--- End diff --
handled in #2450
---
