Repository: spark Updated Branches: refs/heads/master b9994ad05 -> 52fa45d62
[SPARK-17186][SQL] remove catalog table type INDEX ## What changes were proposed in this pull request? Actually Spark SQL doesn't support index, the catalog table type `INDEX` is from Hive. However, most operations in Spark SQL can't handle index table, e.g. create table, alter table, etc. Logically index table should be invisible to end users, and Hive also generates special table name for index table to avoid users accessing it directly. Hive has special SQL syntax to create/show/drop index tables. At Spark SQL side, although we can describe index table directly, but the result is unreadable, we should use the dedicated SQL syntax to do it(e.g. `SHOW INDEX ON tbl`). Spark SQL can also read index table directly, but the result is always empty.(Can hive read index table directly?) This PR remove the table type `INDEX`, to make it clear that Spark SQL doesn't support index currently. ## How was this patch tested? existing tests. Author: Wenchen Fan <wenc...@databricks.com> Closes #14752 from cloud-fan/minor2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/52fa45d6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/52fa45d6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/52fa45d6 Branch: refs/heads/master Commit: 52fa45d62a5a0bc832442f38f9e634c5d8e29e08 Parents: b9994ad Author: Wenchen Fan <wenc...@databricks.com> Authored: Tue Aug 23 23:46:09 2016 -0700 Committer: Reynold Xin <r...@databricks.com> Committed: Tue Aug 23 23:46:09 2016 -0700 ---------------------------------------------------------------------- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 1 - .../org/apache/spark/sql/execution/command/tables.scala | 8 +++----- .../scala/org/apache/spark/sql/hive/MetastoreRelation.scala | 1 - .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 ++-- .../apache/spark/sql/hive/execution/HiveCommandSuite.scala | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/52fa45d6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index f7762e0..83e01f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -200,7 +200,6 @@ case class CatalogTableType private(name: String) object CatalogTableType { val EXTERNAL = new CatalogTableType("EXTERNAL") val MANAGED = new CatalogTableType("MANAGED") - val INDEX = new CatalogTableType("INDEX") val VIEW = new CatalogTableType("VIEW") } http://git-wip-us.apache.org/repos/asf/spark/blob/52fa45d6/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 21544a3..b4a15b8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -620,12 +620,11 @@ case class ShowPartitionsCommand( * Validate and throws an [[AnalysisException]] exception under the following conditions: * 1. If the table is not partitioned. * 2. If it is a datasource table. - * 3. If it is a view or index table. + * 3. If it is a view. */ - if (tab.tableType == VIEW || - tab.tableType == INDEX) { + if (tab.tableType == VIEW) { throw new AnalysisException( - s"SHOW PARTITIONS is not allowed on a view or index table: ${tab.qualifiedName}") + s"SHOW PARTITIONS is not allowed on a view: ${tab.qualifiedName}") } if (tab.partitionColumnNames.isEmpty) { @@ -708,7 +707,6 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman case EXTERNAL => " EXTERNAL TABLE" case VIEW => " VIEW" case MANAGED => " TABLE" - case INDEX => reportUnsupportedError(Seq("index table")) } builder ++= s"CREATE$tableTypeString ${table.quotedString}" http://git-wip-us.apache.org/repos/asf/spark/blob/52fa45d6/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala index 195fce8..d62bc98 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala @@ -80,7 +80,6 @@ private[hive] case class MetastoreRelation( tTable.setTableType(catalogTable.tableType match { case CatalogTableType.EXTERNAL => HiveTableType.EXTERNAL_TABLE.toString case CatalogTableType.MANAGED => HiveTableType.MANAGED_TABLE.toString - case CatalogTableType.INDEX => HiveTableType.INDEX_TABLE.toString case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW.toString }) http://git-wip-us.apache.org/repos/asf/spark/blob/52fa45d6/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 9b7afd4..81d5a12 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -379,8 +379,9 @@ private[hive] class HiveClientImpl( tableType = h.getTableType match { case HiveTableType.EXTERNAL_TABLE => CatalogTableType.EXTERNAL case HiveTableType.MANAGED_TABLE => CatalogTableType.MANAGED - case HiveTableType.INDEX_TABLE => CatalogTableType.INDEX case HiveTableType.VIRTUAL_VIEW => CatalogTableType.VIEW + case HiveTableType.INDEX_TABLE => + throw new AnalysisException("Hive index table is not supported.") }, schema = schema, partitionColumnNames = partCols.map(_.name), @@ -757,7 +758,6 @@ private[hive] class HiveClientImpl( HiveTableType.EXTERNAL_TABLE case CatalogTableType.MANAGED => HiveTableType.MANAGED_TABLE - case CatalogTableType.INDEX => HiveTableType.INDEX_TABLE case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW }) // Note: In Hive the schema and partition columns must be disjoint sets http://git-wip-us.apache.org/repos/asf/spark/blob/52fa45d6/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala index 76aa84b..df33731 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala @@ -424,7 +424,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto val message4 = intercept[AnalysisException] { sql("SHOW PARTITIONS parquet_view1") }.getMessage - assert(message4.contains("is not allowed on a view or index table")) + assert(message4.contains("is not allowed on a view")) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
