[CARBONDATA-1593] Add partition to table cause NoSuchTableException AlterTableSplitCarbonPartition's processSchema method doesn't provide db info to sparkSession.catalog.refreshTable, this will cause NoSuchTableException when we add partitions to carbondata table.
This closes #1452 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9454499 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9454499 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9454499 Branch: refs/heads/pre-aggregate Commit: e9454499dcb89ed69b1e18f79b3003ea8e5d8d25 Parents: 0586146 Author: wyp <wyphao.2...@163.com> Authored: Mon Oct 30 12:49:53 2017 +0800 Committer: lionelcao <whuca...@gmail.com> Committed: Thu Nov 2 10:10:45 2017 +0800 ---------------------------------------------------------------------- .../AlterTableSplitCarbonPartitionCommand.scala | 3 +- .../partition/TestAlterPartitionTable.scala | 57 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9454499/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala index e16dfc9..12bf31e 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala @@ -23,6 +23,7 @@ import java.util import scala.collection.JavaConverters._ import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.command.{AlterTableSplitPartitionModel, DataProcessCommand, RunnableCommand, SchemaProcessCommand} import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonRelation} import org.apache.spark.util.{AlterTableUtil, PartitionUtils} @@ -109,7 +110,7 @@ case class AlterTableSplitCarbonPartitionCommand( CarbonUtil.writeThriftTableToSchemaFile(schemaFilePath, thriftTable) // update the schema modified time carbonMetaStore.updateAndTouchSchemasUpdatedTime(storePath) - sparkSession.catalog.refreshTable(tableName) + sparkSession.sessionState.catalog.refreshTable(TableIdentifier(tableName, Option(dbName))) Seq.empty } http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9454499/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala index 5e81044..04de9a3 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala @@ -779,6 +779,63 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll { .contains("Data in range info must be the same type with the partition field's type")) } + test("Add partition to table in or not in default database") { + sql("DROP TABLE IF EXISTS carbon_table_default_db") + sql( + """ + | CREATE TABLE carbon_table_default_db(id INT, name STRING) PARTITIONED BY (dt STRING) + | STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016') + """.stripMargin) + sql("ALTER TABLE carbon_table_default_db ADD PARTITION ('2017')") + + val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_carbon_table_default_db") + val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName) + val partitionIds = partitionInfo.getPartitionIds + val range_info = partitionInfo.getRangeInfo + assert(partitionIds == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava) + assert(partitionInfo.getMaxPartitionId == 3) + assert(partitionInfo.getNumPartitions == 4) + assert(range_info.get(0) == "2015") + assert(range_info.get(1) == "2016") + assert(range_info.get(2) == "2017") + + sql("CREATE DATABASE IF NOT EXISTS carbondb") + sql("DROP TABLE IF EXISTS carbondb.carbontable") + sql( + """ + | CREATE TABLE carbondb.carbontable(id INT, name STRING) PARTITIONED BY (dt STRING) + | STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016') + """.stripMargin) + sql("ALTER TABLE carbondb.carbontable ADD PARTITION ('2017')") + + val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("carbondb_carbontable") + val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable1.getFactTableName) + val partitionIds1 = partitionInfo1.getPartitionIds + val range_info1 = partitionInfo1.getRangeInfo + assert(partitionIds1 == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava) + assert(partitionInfo1.getMaxPartitionId == 3) + assert(partitionInfo1.getNumPartitions == 4) + assert(range_info1.get(0) == "2015") + assert(range_info1.get(1) == "2016") + assert(range_info1.get(2) == "2017") + } + + test("test exception when alter partition's table doesn't exist in a perticular database") { + val exception_test_add_partition: Exception = intercept[Exception] { + sql("CREATE DATABASE IF NOT EXISTS carbondb") + sql("USE default") + sql( + """ + | CREATE TABLE carbon_table_in_default_db(id INT, name STRING) + | PARTITIONED BY (dt STRING) STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2015,2016') + """.stripMargin) + sql("ALTER TABLE carbondb.carbon_table_in_default_db ADD PARTITION ('2017')") + } + assert(exception_test_add_partition.getMessage + .contains("Table or view 'carbon_table_in_default_db' not found in database 'carbondb'")) + } + def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Seq[Int]): Unit = { val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName) val dataFiles = getDataFiles(carbonTable, segmentId)