This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c7d63eac48d [SPARK-44873] Support alter view with nested columns in
Hive client
c7d63eac48d is described below
commit c7d63eac48d3a81099456360f1c30e6049824749
Author: kylerong-db <kyle.r...@databricks.com>
AuthorDate: Fri Aug 18 11:07:21 2023 -0700
[SPARK-44873] Support alter view with nested columns in Hive client
### What changes were proposed in this pull request?
Previously, if a view's schema contains a nested struct, alterTable using
Hive client will fail. This change supports a view with a nested struct. The
mechanism is to store an empty schema when we call Hive client, since we
already store the actual schema in table properties. This fix is similar to
https://github.com/apache/spark/pull/37364
### Why are the changes needed?
This supports using view with nested structs in Hive metastore.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added unit test.
Closes #42532 from kylerong-db/hive_view.
Authored-by: kylerong-db <kyle.r...@databricks.com>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
.../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 11 ++++++++++-
.../org/apache/spark/sql/hive/HiveParquetSourceSuite.scala | 10 ++++++++++
.../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 8 --------
3 files changed, 20 insertions(+), 9 deletions(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index b1eadea42e0..67b780f13c4 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -595,7 +595,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf,
hadoopConf: Configurat
if (tableDefinition.tableType == VIEW) {
val newTableProps = tableDefinition.properties ++
tableMetaToTableProps(tableDefinition).toMap
- client.alterTable(tableDefinition.copy(properties = newTableProps))
+ val newTable = tableDefinition.copy(properties = newTableProps)
+ try {
+ client.alterTable(newTable)
+ } catch {
+ case NonFatal(e) =>
+ // If for some reason we fail to store the schema we store it as
empty there
+ // since we already store the real schema in the table properties.
This try-catch
+ // should only be necessary for Spark views which are incompatible
with Hive
+ client.alterTable(newTable.copy(schema = EMPTY_DATA_SCHEMA))
+ }
} else {
val oldTableDef = getRawTable(db, tableDefinition.identifier.table)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
index 7c67f34560e..45668fc683d 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
@@ -385,4 +385,14 @@ class HiveParquetSourceSuite extends
ParquetPartitioningTest with ParquetTest {
checkAnswer(spark.table("t"), Row(Row("a", 1)))
}
}
+
+ test("Alter view with nested struct") {
+ withView("t", "t2") {
+ sql("CREATE OR REPLACE VIEW t AS SELECT " +
+ "struct(id AS `$col2`, struct(id AS `$col`) AS s1) AS s2 FROM
RANGE(5)")
+ sql("ALTER VIEW t SET TBLPROPERTIES ('x' = 'y')")
+ sql("ALTER VIEW t RENAME TO t2")
+ checkAnswer(sql("show TBLPROPERTIES t2 (x)"), Row("x", "y"))
+ }
+ }
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 1c46b558708..201ba5ea6a1 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -175,14 +175,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with
TestHiveSingleton with BeforeA
withView("v") {
spark.sql("CREATE VIEW v AS SELECT STRUCT('a' AS `a`, 1 AS b) q")
checkAnswer(sql("SELECT q.`a`, q.b FROM v"), Row("a", 1) :: Nil)
-
- checkError(
- exception = intercept[SparkException] {
- spark.sql("ALTER VIEW v AS SELECT STRUCT('a' AS `$a`, 1 AS b) q")
- },
- errorClass = "CANNOT_RECOGNIZE_HIVE_TYPE",
- parameters = Map("fieldType" -> "\"STRUCT<$A:STRING,B:INT>\"",
"fieldName" -> "`q`")
- )
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
