This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new c7d63eac48d [SPARK-44873] Support alter view with nested columns in Hive client c7d63eac48d is described below commit c7d63eac48d3a81099456360f1c30e6049824749 Author: kylerong-db <kyle.r...@databricks.com> AuthorDate: Fri Aug 18 11:07:21 2023 -0700 [SPARK-44873] Support alter view with nested columns in Hive client ### What changes were proposed in this pull request? Previously, if a view's schema contains a nested struct, alterTable using Hive client will fail. This change supports a view with a nested struct. The mechanism is to store an empty schema when we call Hive client, since we already store the actual schema in table properties. This fix is similar to https://github.com/apache/spark/pull/37364 ### Why are the changes needed? This supports using view with nested structs in Hive metastore. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added unit test. Closes #42532 from kylerong-db/hive_view. Authored-by: kylerong-db <kyle.r...@databricks.com> Signed-off-by: Gengliang Wang <gengli...@apache.org> --- .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 11 ++++++++++- .../org/apache/spark/sql/hive/HiveParquetSourceSuite.scala | 10 ++++++++++ .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 8 -------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index b1eadea42e0..67b780f13c4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -595,7 +595,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat if (tableDefinition.tableType == VIEW) { val newTableProps = tableDefinition.properties ++ tableMetaToTableProps(tableDefinition).toMap - client.alterTable(tableDefinition.copy(properties = newTableProps)) + val newTable = tableDefinition.copy(properties = newTableProps) + try { + client.alterTable(newTable) + } catch { + case NonFatal(e) => + // If for some reason we fail to store the schema we store it as empty there + // since we already store the real schema in the table properties. This try-catch + // should only be necessary for Spark views which are incompatible with Hive + client.alterTable(newTable.copy(schema = EMPTY_DATA_SCHEMA)) + } } else { val oldTableDef = getRawTable(db, tableDefinition.identifier.table) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala index 7c67f34560e..45668fc683d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala @@ -385,4 +385,14 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest with ParquetTest { checkAnswer(spark.table("t"), Row(Row("a", 1))) } } + + test("Alter view with nested struct") { + withView("t", "t2") { + sql("CREATE OR REPLACE VIEW t AS SELECT " + + "struct(id AS `$col2`, struct(id AS `$col`) AS s1) AS s2 FROM RANGE(5)") + sql("ALTER VIEW t SET TBLPROPERTIES ('x' = 'y')") + sql("ALTER VIEW t RENAME TO t2") + checkAnswer(sql("show TBLPROPERTIES t2 (x)"), Row("x", "y")) + } + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 1c46b558708..201ba5ea6a1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -175,14 +175,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA withView("v") { spark.sql("CREATE VIEW v AS SELECT STRUCT('a' AS `a`, 1 AS b) q") checkAnswer(sql("SELECT q.`a`, q.b FROM v"), Row("a", 1) :: Nil) - - checkError( - exception = intercept[SparkException] { - spark.sql("ALTER VIEW v AS SELECT STRUCT('a' AS `$a`, 1 AS b) q") - }, - errorClass = "CANNOT_RECOGNIZE_HIVE_TYPE", - parameters = Map("fieldType" -> "\"STRUCT<$A:STRING,B:INT>\"", "fieldName" -> "`q`") - ) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org