This is an automated email from the ASF dual-hosted git repository.
lixiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d1937c1 [SPARK-27926][SQL] Allow altering table add columns with
CSVFileFormat/JsonFileFormat provider
d1937c1 is described below
commit d1937c14795aed4bea89065c3796a5bc1dd18c8f
Author: Gengliang Wang <gengliang.w...@databricks.com>
AuthorDate: Mon Jun 3 23:51:05 2019 -0700
[SPARK-27926][SQL] Allow altering table add columns with
CSVFileFormat/JsonFileFormat provider
## What changes were proposed in this pull request?
In the previous work of csv/json migration, CSVFileFormat/JsonFileFormat is
removed in the table provider whitelist of
`AlterTableAddColumnsCommand.verifyAlterTableAddColumn`:
https://github.com/apache/spark/pull/24005
https://github.com/apache/spark/pull/24058
This is regression. If a table is created with Provider
`org.apache.spark.sql.execution.datasources.csv.CSVFileFormat` or
`org.apache.spark.sql.execution.datasources.json.JsonFileFormat`, Spark should
allow the "alter table add column" operation.
## How was this patch tested?
Unit test
Closes #24776 from gengliangwang/v1Table.
Authored-by: Gengliang Wang <gengliang.w...@databricks.com>
Signed-off-by: gatorsmile <gatorsm...@gmail.com>
---
.../main/scala/org/apache/spark/sql/execution/command/tables.scala | 5 ++++-
.../test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala | 5 ++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index ea29eff..64f739f 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -37,6 +37,8 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString,
quoteIdentifier}
import org.apache.spark.sql.execution.datasources.{DataSource,
PartitioningUtils}
+import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat
+import org.apache.spark.sql.execution.datasources.json.JsonFileFormat
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.execution.datasources.v2.csv.CSVDataSourceV2
import org.apache.spark.sql.execution.datasources.v2.json.JsonDataSourceV2
@@ -238,7 +240,8 @@ case class AlterTableAddColumnsCommand(
// TextFileFormat only default to one column "value"
// Hive type is already considered as hive serde table, so the logic
will not
// come in here.
- case _: JsonDataSourceV2 | _: CSVDataSourceV2 | _: ParquetFileFormat |
_: OrcDataSourceV2 =>
+ case _: CSVFileFormat | _: JsonFileFormat | _: ParquetFileFormat =>
+ case _: JsonDataSourceV2 | _: CSVDataSourceV2 | _: OrcDataSourceV2 =>
case s if s.getClass.getCanonicalName.endsWith("OrcFileFormat") =>
case s =>
throw new AnalysisException(
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 0124f28..b777db7 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2566,7 +2566,10 @@ abstract class DDLSuite extends QueryTest with
SQLTestUtils {
}
}
- val supportedNativeFileFormatsForAlterTableAddColumns = Seq("parquet",
"json", "csv")
+ val supportedNativeFileFormatsForAlterTableAddColumns = Seq("csv", "json",
"parquet",
+ "org.apache.spark.sql.execution.datasources.csv.CSVFileFormat",
+ "org.apache.spark.sql.execution.datasources.json.JsonFileFormat",
+ "org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat")
supportedNativeFileFormatsForAlterTableAddColumns.foreach { provider =>
test(s"alter datasource table add columns - $provider") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
