Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13386#discussion_r65265253
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala ---
@@ -936,7 +936,39 @@ class SparkSqlAstBuilder(conf: SQLConf) extends
AstBuilder {
comment = comment)
selectQuery match {
- case Some(q) => CreateTableAsSelectLogicalPlan(tableDesc, q,
ifNotExists)
+ case Some(q) =>
+ // Hive does not allow to use a CTAS statement to create a
partitioned table.
+ if (tableDesc.partitionColumnNames.nonEmpty) {
+ val errorMessage = "A Create Table As Select (CTAS) statement is
not allowed to " +
+ "create a partitioned table using Hive's file formats. " +
+ "Please use the syntax of \"CREATE TABLE tableName USING
dataSource " +
+ "OPTIONS (...) PARTITIONED BY ...\" to create a partitioned
table through a " +
+ "CTAS statement."
+ throw operationNotAllowed(errorMessage, ctx)
+ }
+
+ val hasStorageProperties = (ctx.createFileFormat != null) ||
(ctx.rowFormat != null)
+ if (conf.convertCTAS && !hasStorageProperties) {
+ val mode = if (ifNotExists) SaveMode.Ignore else
SaveMode.ErrorIfExists
+ val options = rowStorage.serdeProperties ++
fileStorage.serdeProperties
+ val optionsWithPath = if (location.isDefined) {
+ options + ("path" -> location.get)
+ } else {
+ options
+ }
+ CreateTableUsingAsSelect(
+ tableIdent = tableDesc.identifier,
+ provider = conf.defaultDataSourceName,
+ temporary = false,
+ partitionColumns = tableDesc.partitionColumnNames.toArray,
+ bucketSpec = None,
+ mode = mode,
+ options = optionsWithPath,
+ q
+ )
+ } else {
+ CreateTableAsSelectLogicalPlan(tableDesc, q, ifNotExists)
--- End diff --
Looking at its implementation, it is not hive-specific. So, seems fine to
leave it as is.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
