This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new cb8de35b749 [SPARK-44873] Support alter view with nested columns in
Hive client
cb8de35b749 is described below
commit cb8de35b749e8e2ff3d0667be3e933c682c528ee
Author: kylerong-db <kyle.r...@databricks.com>
AuthorDate: Fri Aug 18 11:07:21 2023 -0700
[SPARK-44873] Support alter view with nested columns in Hive client
### What changes were proposed in this pull request?
Previously, if a view's schema contains a nested struct, alterTable using
Hive client will fail. This change supports a view with a nested struct. The
mechanism is to store an empty schema when we call Hive client, since we
already store the actual schema in table properties. This fix is similar to
https://github.com/apache/spark/pull/37364
### Why are the changes needed?
This supports using view with nested structs in Hive metastore.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added unit test.
Closes #42532 from kylerong-db/hive_view.
Authored-by: kylerong-db <kyle.r...@databricks.com>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
(cherry picked from commit c7d63eac48d3a81099456360f1c30e6049824749)
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
.../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 11 ++++++++++-
.../org/apache/spark/sql/hive/HiveParquetSourceSuite.scala | 10 ++++++++++
.../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 8 --------
3 files changed, 20 insertions(+), 9 deletions(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index b1eadea42e0..67b780f13c4 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -595,7 +595,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf,
hadoopConf: Configurat
if (tableDefinition.tableType == VIEW) {
val newTableProps = tableDefinition.properties ++
tableMetaToTableProps(tableDefinition).toMap
- client.alterTable(tableDefinition.copy(properties = newTableProps))
+ val newTable = tableDefinition.copy(properties = newTableProps)
+ try {
+ client.alterTable(newTable)
+ } catch {
+ case NonFatal(e) =>
+ // If for some reason we fail to store the schema we store it as
empty there
+ // since we already store the real schema in the table properties.
This try-catch
+ // should only be necessary for Spark views which are incompatible
with Hive
+ client.alterTable(newTable.copy(schema = EMPTY_DATA_SCHEMA))
+ }
} else {
val oldTableDef = getRawTable(db, tableDefinition.identifier.table)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
index 7c67f34560e..45668fc683d 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
@@ -385,4 +385,14 @@ class HiveParquetSourceSuite extends
ParquetPartitioningTest with ParquetTest {
checkAnswer(spark.table("t"), Row(Row("a", 1)))
}
}
+
+ test("Alter view with nested struct") {
+ withView("t", "t2") {
+ sql("CREATE OR REPLACE VIEW t AS SELECT " +
+ "struct(id AS `$col2`, struct(id AS `$col`) AS s1) AS s2 FROM
RANGE(5)")
+ sql("ALTER VIEW t SET TBLPROPERTIES ('x' = 'y')")
+ sql("ALTER VIEW t RENAME TO t2")
+ checkAnswer(sql("show TBLPROPERTIES t2 (x)"), Row("x", "y"))
+ }
+ }
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 006caa02d55..2c5e2956f5f 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -175,14 +175,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with
TestHiveSingleton with BeforeA
withView("v") {
spark.sql("CREATE VIEW v AS SELECT STRUCT('a' AS `a`, 1 AS b) q")
checkAnswer(sql("SELECT q.`a`, q.b FROM v"), Row("a", 1) :: Nil)
-
- checkError(
- exception = intercept[SparkException] {
- spark.sql("ALTER VIEW v AS SELECT STRUCT('a' AS `$a`, 1 AS b) q")
- },
- errorClass = "CANNOT_RECOGNIZE_HIVE_TYPE",
- parameters = Map("fieldType" -> "\"STRUCT<$A:STRING,B:INT>\"",
"fieldName" -> "`q`")
- )
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
