Repository: spark Updated Branches: refs/heads/branch-1.6 abe8f991a -> a57f87ee4
[SPARK-13454][SQL] Allow users to drop a table with a name starting with an underscore. ## What changes were proposed in this pull request? This change adds a workaround to allow users to drop a table with a name starting with an underscore. Without this patch, we can create such a table, but we cannot drop it. The reason is that Hive's parser unquote an quoted identifier (see https://github.com/apache/hive/blob/release-1.2.1/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g#L453). So, when we issue a drop table command to Hive, a table name starting with an underscore is actually not quoted. Then, Hive will complain about it because it does not support a table name starting with an underscore without using backticks (underscores are allowed as long as it is not the first char though). ## How was this patch tested? Add a test to make sure we can drop a table with a name starting with an underscore. https://issues.apache.org/jira/browse/SPARK-13454 Author: Yin Huai <yh...@databricks.com> Closes #11349 from yhuai/fixDropTable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a57f87ee Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a57f87ee Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a57f87ee Branch: refs/heads/branch-1.6 Commit: a57f87ee4aafdb97c15f4076e20034ea34c7e2e5 Parents: abe8f99 Author: Yin Huai <yh...@databricks.com> Authored: Fri Feb 26 12:34:03 2016 -0800 Committer: Yin Huai <yh...@databricks.com> Committed: Fri Feb 26 12:34:03 2016 -0800 ---------------------------------------------------------------------- .../spark/sql/hive/execution/commands.scala | 20 +++++++++++++++++-- .../sql/hive/HiveMetastoreCatalogSuite.scala | 21 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a57f87ee/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala index 94210a5..6b16d59 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution import org.apache.hadoop.hive.metastore.MetaStoreUtils import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.{SqlParser, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.EliminateSubQueries import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -70,7 +70,23 @@ case class DropTable( case e: Throwable => log.warn(s"${e.getMessage}", e) } hiveContext.invalidateTable(tableName) - hiveContext.runSqlHive(s"DROP TABLE $ifExistsClause$tableName") + val tableNameForHive = { + // Hive's parser will unquote an identifier (see the rule of QuotedIdentifier in + // HiveLexer.g of Hive 1.2.1). For the DROP TABLE command that we pass in Hive, we + // will use the quoted form (db.tableName) if the table name starts with a _. + // Otherwise, we keep the unquoted form (`db`.`tableName`), which is the same as tableName + // passed into this DropTable class. Please note that although QuotedIdentifier rule + // allows backticks appearing in an identifier, Hive does not actually allow such + // an identifier be a table name. So, we do not check if a table name part has + // any backtick or not. + // + // This change is at here because this patch is just for 1.6 branch and we try to + // avoid of affecting normal cases (tables do not use _ as the first character of + // their name). + val identifier = SqlParser.parseTableIdentifier(tableName) + if (identifier.table.startsWith("_")) identifier.quotedString else identifier.unquotedString + } + hiveContext.runSqlHive(s"DROP TABLE $ifExistsClause$tableNameForHive") hiveContext.catalog.unregisterTable(TableIdentifier(tableName)) Seq.empty[Row] } http://git-wip-us.apache.org/repos/asf/spark/blob/a57f87ee/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala index d63f3d3..15fd96a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala @@ -47,6 +47,27 @@ class HiveMetastoreCatalogSuite extends SparkFunSuite with TestHiveSingleton { logInfo(df.queryExecution.toString) df.as('a).join(df.as('b), $"a.key" === $"b.key") } + + test("SPARK-13454: drop a table with a name starting with underscore") { + hiveContext.range(10).write.saveAsTable("_spark13454") + hiveContext.range(20).registerTempTable("_spark13454") + // This will drop both metastore table and temp table. + hiveContext.sql("drop table `_spark13454`") + assert(hiveContext.tableNames().filter(name => name == "_spark13454").length === 0) + + hiveContext.range(10).write.saveAsTable("_spark13454") + hiveContext.range(20).registerTempTable("_spark13454") + hiveContext.sql("drop table default.`_spark13454`") + // This will drop the metastore table but keep the temptable. + assert(hiveContext.tableNames().filter(name => name == "_spark13454").length === 1) + // Make sure it is the temp table. + assert(hiveContext.table("_spark13454").count() === 20) + hiveContext.sql("drop table if exists `_spark13454`") + assert(hiveContext.tableNames().filter(name => name == "_spark13454").length === 0) + + hiveContext.range(10).write.saveAsTable("spark13454") + hiveContext.sql("drop table spark13454") + } } class DataSourceWithHiveMetastoreCatalogSuite --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org