This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 3edcf37 [SPARK-34421][SQL][3.0] Resolve temporary functions and views
in views with CTEs
3edcf37 is described below
commit 3edcf379be8f3a84471e32c14120cc3f1a6edace
Author: Peter Toth <peter.t...@gmail.com>
AuthorDate: Mon Feb 22 04:42:02 2021 +0000
[SPARK-34421][SQL][3.0] Resolve temporary functions and views in views with
CTEs
### What changes were proposed in this pull request?
This PR:
- Fixes a bug that doesn't report analysis error when it should:
```
CREATE TEMPORARY VIEW temp_view AS SELECT 0;
CREATE VIEW view_on_temp_view AS WITH cte AS (SELECT * FROM temp_view)
SELECT * FROM cte
```
by properly checking temporary objects in view definitions with CTEs.
### Why are the changes needed?
To fix a bug.
### Does this PR introduce _any_ user-facing change?
Yes, analysis error is thrown as expected.
### How was this patch tested?
Added new UT + existing ones.
Closes #31592 from
peter-toth/SPARK-34421-temp-functions-in-views-with-cte-3.0.
Authored-by: Peter Toth <peter.t...@gmail.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../apache/spark/sql/execution/command/views.scala | 3 +-
.../scala/org/apache/spark/sql/SQLQuerySuite.scala | 40 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 991ebc3..ebba726 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -25,7 +25,7 @@ import
org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, Pe
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat,
CatalogTable, CatalogTableType, SessionCatalog}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute,
AttributeReference, SubqueryExpression}
import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project,
View, With}
import
org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
import org.apache.spark.sql.internal.StaticSQLConf
import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType}
@@ -180,6 +180,7 @@ case class CreateViewCommand(
throw new AnalysisException(s"Not allowed to create a permanent
view $name by " +
s"referencing a temporary view ${nameParts.quoted}. " +
"Please create a temp view instead by CREATE TEMP VIEW")
+ case w: With if !w.resolved => w.innerChildren.foreach(verify)
case other if !other.resolved => other.expressions.flatMap(_.collect
{
// Traverse subquery plan for any unresolved relations.
case e: SubqueryExpression => verify(e.plan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 409e645..88524d7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3650,6 +3650,46 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
}
}
}
+
+ test("SPARK-34421: Resolve temporary objects in permanent views with CTEs") {
+ val tempFuncName = "temp_func"
+ withUserDefinedFunction((tempFuncName, true)) {
+ spark.udf.register(tempFuncName, identity[Int](_))
+
+ val tempViewName = "temp_view"
+ withTempView(tempViewName) {
+ sql(s"CREATE TEMPORARY VIEW $tempViewName AS SELECT 1")
+
+ val testViewName = "test_view"
+
+ val e = intercept[AnalysisException] {
+ sql(
+ s"""
+ |CREATE VIEW $testViewName AS
+ |WITH cte AS (
+ | SELECT * FROM $tempViewName
+ |)
+ |SELECT * FROM cte
+ |""".stripMargin)
+ }
+ assert(e.message.contains("Not allowed to create a permanent view " +
+ s"`default`.`$testViewName` by referencing a temporary view
$tempViewName"))
+
+ val e2 = intercept[AnalysisException] {
+ sql(
+ s"""
+ |CREATE VIEW $testViewName AS
+ |WITH cte AS (
+ | SELECT $tempFuncName(0)
+ |)
+ |SELECT * FROM cte
+ |""".stripMargin)
+ }
+ assert(e2.message.contains("Not allowed to create a permanent view " +
+ s"`default`.`$testViewName` by referencing a temporary function
`$tempFuncName`"))
+ }
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
