This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 75b902f [SPARK-23519][SQL][2.4] Create view should work from query
with duplicate output columns
75b902f is described below
commit 75b902f547dfd392f17013210a03dc671f94fcdc
Author: hemanth meka <[email protected]>
AuthorDate: Tue Sep 10 10:52:03 2019 +0800
[SPARK-23519][SQL][2.4] Create view should work from query with duplicate
output columns
**What changes were proposed in this pull request?**
Backporting the pullrequest
[25570](https://github.com/apache/spark/pull/25570) to branch-2.4
Moving the call for checkColumnNameDuplication out of
generateViewProperties. This way we can choose ifcheckColumnNameDuplication
will be performed on analyzed or aliased plan without having to pass an
additional argument(aliasedPlan) to generateViewProperties.
Before the pr column name duplication was performed on the query output of
below sql(c1, c1) and the pr makes it perform check on the user provided schema
of view definition(c1, c2)
**Why are the changes needed?**
Changes are to fix SPARK-23519 bug. Below queries would cause an exception.
This pr fixes them and also added a test case.
`CREATE TABLE t23519 AS SELECT 1 AS c1 CREATE VIEW v23519 (c1, c2) AS
SELECT c1, c1 FROM t23519`
Does this PR introduce any user-facing change?
No
**How was this patch tested?**
new unit test added in SQLViewSuite
Closes #25733 from hem1891/SPARK-23519-backport-to-2.4.
Lead-authored-by: hemanth meka <[email protected]>
Co-authored-by: hem1891 <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/execution/command/views.scala | 18 +++++++++++-------
.../org/apache/spark/sql/execution/SQLViewSuite.scala | 10 ++++++++++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 5172f32..abc8515 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -26,7 +26,7 @@ import
org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
-import org.apache.spark.sql.types.MetadataBuilder
+import org.apache.spark.sql.types.{MetadataBuilder, StructType}
import org.apache.spark.sql.util.SchemaUtils
@@ -233,14 +233,15 @@ case class CreateViewCommand(
throw new AnalysisException(
"It is not allowed to create a persisted view from the Dataset API")
}
-
- val newProperties = generateViewProperties(properties, session,
analyzedPlan)
+ val aliasedSchema = aliasPlan(session, analyzedPlan).schema
+ val newProperties = generateViewProperties(
+ properties, session, analyzedPlan, aliasedSchema.fieldNames)
CatalogTable(
identifier = name,
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
- schema = aliasPlan(session, analyzedPlan).schema,
+ schema = aliasedSchema,
properties = newProperties,
viewText = originalText,
comment = comment
@@ -294,7 +295,8 @@ case class AlterViewAsCommand(
val viewIdent = viewMeta.identifier
checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
- val newProperties = generateViewProperties(viewMeta.properties, session,
analyzedPlan)
+ val newProperties = generateViewProperties(
+ viewMeta.properties, session, analyzedPlan,
analyzedPlan.schema.fieldNames)
val updatedViewMeta = viewMeta.copy(
schema = analyzedPlan.schema,
@@ -355,13 +357,15 @@ object ViewHelper {
def generateViewProperties(
properties: Map[String, String],
session: SparkSession,
- analyzedPlan: LogicalPlan): Map[String, String] = {
+ analyzedPlan: LogicalPlan,
+ fieldNames: Array[String]): Map[String, String] = {
+ // for createViewCommand queryOutput may be different from fieldNames
val queryOutput = analyzedPlan.schema.fieldNames
// Generate the query column names, throw an AnalysisException if there
exists duplicate column
// names.
SchemaUtils.checkColumnNameDuplication(
- queryOutput, "in the view definition",
session.sessionState.conf.resolver)
+ fieldNames, "in the view definition", session.sessionState.conf.resolver)
// Generate the view default database name.
val viewDefaultDatabase = session.sessionState.catalog.getCurrentDatabase
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
index 8269d4d..d91c451 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
@@ -706,4 +706,14 @@ abstract class SQLViewSuite extends QueryTest with
SQLTestUtils {
}
}
}
+
+ test("SPARK-23519 view should be created even when query output contains
duplicate col name") {
+ withTable("t23519") {
+ withView("v23519") {
+ sql("CREATE TABLE t23519 USING parquet AS SELECT 1 AS c1")
+ sql("CREATE VIEW v23519 (c1, c2) AS SELECT c1, c1 FROM t23519")
+ checkAnswer(sql("SELECT * FROM v23519"), Row(1, 1))
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
