This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 4fcb5ae [SPARK-31024][SQL] Allow specifying session catalog name `spark_catalog` in qualified column names for v1 tables 4fcb5ae is described below commit 4fcb5ae22623de96247fcfe7341be0af7ed2471f Author: Terry Kim <yumin...@gmail.com> AuthorDate: Thu Mar 5 18:33:59 2020 +0800 [SPARK-31024][SQL] Allow specifying session catalog name `spark_catalog` in qualified column names for v1 tables ### What changes were proposed in this pull request? Currently, the user cannot specify the session catalog name (`spark_catalog`) in qualified column names for v1 tables: ``` SELECT spark_catalog.default.t.i FROM spark_catalog.default.t ``` fails with `cannot resolve 'spark_catalog.default.t.i`. This is inconsistent with v2 table behavior where catalog name can be used: ``` SELECT testcat.ns1.tbl.id FROM testcat.ns1.tbl.id ``` This PR proposes to fix the inconsistency and allow the user to specify session catalog name in column names for v1 tables. ### Why are the changes needed? Fixing an inconsistent behavior. ### Does this PR introduce any user-facing change? Yes, now the following query works: ``` SELECT spark_catalog.default.t.i FROM spark_catalog.default.t ``` ### How was this patch tested? Added new tests. Closes #27776 from imback82/spark_catalog_col_name_resolution. Authored-by: Terry Kim <yumin...@gmail.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit 66b4fd040e97cb6de6536a5545017278879c98fb) Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../spark/sql/catalyst/analysis/Analyzer.scala | 2 +- .../sql/catalyst/catalog/SessionCatalog.scala | 6 +- .../spark/sql/catalyst/expressions/package.scala | 102 ++++++++++++++++----- .../sql/catalyst/catalog/SessionCatalogSuite.scala | 6 +- .../results/columnresolution-negative.sql.out | 26 +++--- .../results/postgreSQL/create_view.sql.out | 2 +- .../sql-tests/results/postgreSQL/join.sql.out | 2 +- .../results/postgreSQL/select_having.sql.out | 2 +- .../results/postgreSQL/window_part3.sql.out | 4 +- .../results/udf/postgreSQL/udf-join.sql.out | 2 +- .../udf/postgreSQL/udf-select_having.sql.out | 2 +- .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 +- .../spark/sql/connector/DataSourceV2SQLSuite.scala | 8 +- .../spark/sql/hive/HiveMetastoreCatalogSuite.scala | 2 +- 14 files changed, 111 insertions(+), 59 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 254dd44..3cb754d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -935,7 +935,7 @@ class Analyzer( v1SessionCatalog.getRelation(v1Table.v1Table) case table => SubqueryAlias( - ident.asMultipartIdentifier, + catalog.name +: ident.asMultipartIdentifier, DataSourceV2Relation.create(table, Some(catalog), Some(ident))) } val key = catalog.name +: ident.namespace :+ ident.name diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index c80d9d2..3a63aff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, Im import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View} import org.apache.spark.sql.catalyst.util.StringUtils +import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE import org.apache.spark.sql.types.StructType @@ -758,6 +759,7 @@ class SessionCatalog( val name = metadata.identifier val db = formatDatabaseName(name.database.getOrElse(currentDb)) val table = formatTableName(name.table) + val multiParts = Seq(CatalogManager.SESSION_CATALOG_NAME, db, table) if (metadata.tableType == CatalogTableType.VIEW) { val viewText = metadata.viewText.getOrElse(sys.error("Invalid view without text.")) @@ -769,9 +771,9 @@ class SessionCatalog( desc = metadata, output = metadata.schema.toAttributes, child = parser.parsePlan(viewText)) - SubqueryAlias(table, db, child) + SubqueryAlias(multiParts, child) } else { - SubqueryAlias(table, db, UnresolvedCatalogRelation(metadata)) + SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata)) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala index 9f42e64..1b59056 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala @@ -142,22 +142,44 @@ package object expressions { } /** Map to use for qualified case insensitive attribute lookups with 3 part key */ - @transient private val qualified3Part: Map[(String, String, String), Seq[Attribute]] = { + @transient private lazy val qualified3Part: Map[(String, String, String), Seq[Attribute]] = { // key is 3 part: database name, table name and name - val grouped = attrs.filter(_.qualifier.length == 2).groupBy { a => - (a.qualifier.head.toLowerCase(Locale.ROOT), - a.qualifier.last.toLowerCase(Locale.ROOT), - a.name.toLowerCase(Locale.ROOT)) + val grouped = attrs.filter(a => a.qualifier.length >= 2 && a.qualifier.length <= 3) + .groupBy { a => + val qualifier = if (a.qualifier.length == 2) { + a.qualifier + } else { + a.qualifier.takeRight(2) + } + (qualifier.head.toLowerCase(Locale.ROOT), + qualifier.last.toLowerCase(Locale.ROOT), + a.name.toLowerCase(Locale.ROOT)) + } + unique(grouped) + } + + /** Map to use for qualified case insensitive attribute lookups with 4 part key */ + @transient + private lazy val qualified4Part: Map[(String, String, String, String), Seq[Attribute]] = { + // key is 4 part: catalog name, database name, table name and name + val grouped = attrs.filter(_.qualifier.length == 3).groupBy { a => + a.qualifier match { + case Seq(catalog, db, tbl) => + (catalog.toLowerCase(Locale.ROOT), + db.toLowerCase(Locale.ROOT), + tbl.toLowerCase(Locale.ROOT), + a.name.toLowerCase(Locale.ROOT)) + } } unique(grouped) } - /** Returns true if all qualifiers in `attrs` have 2 or less parts. */ - @transient private val hasTwoOrLessQualifierParts: Boolean = - attrs.forall(_.qualifier.length <= 2) + /** Returns true if all qualifiers in `attrs` have 3 or less parts. */ + @transient private val hasThreeOrLessQualifierParts: Boolean = + attrs.forall(_.qualifier.length <= 3) - /** Match attributes for the case where all qualifiers in `attrs` have 2 or less parts. */ - private def matchWithTwoOrLessQualifierParts( + /** Match attributes for the case where all qualifiers in `attrs` have 3 or less parts. */ + private def matchWithThreeOrLessQualifierParts( nameParts: Seq[String], resolver: Resolver): (Seq[Attribute], Seq[String]) = { // Collect matching attributes given a name and a lookup. @@ -167,25 +189,55 @@ package object expressions { } } - // Find matches for the given name assuming that the 1st two parts are qualifier - // (i.e. database name and table name) and the 3rd part is the actual column name. + // Find matches for the given name assuming that the 1st three parts are qualifier + // (i.e. catalog name, database name and table name) and the 4th part is the actual + // column name. // - // For example, consider an example where "db1" is the database name, "a" is the table name - // and "b" is the column name and "c" is the struct field name. - // If the name parts is db1.a.b.c, then Attribute will match - // Attribute(b, qualifier("db1,"a")) and List("c") will be the second element + // For example, consider an example where "cat" is the catalog name, "db1" is the database + // name, "a" is the table name and "b" is the column name and "c" is the struct field name. + // If the name parts is cat.db1.a.b.c, then Attribute will match + // Attribute(b, qualifier("cat", "db1, "a")) and List("c") will be the second element var matches: (Seq[Attribute], Seq[String]) = nameParts match { - case dbPart +: tblPart +: name +: nestedFields => - val key = (dbPart.toLowerCase(Locale.ROOT), + case catalogPart +: dbPart +: tblPart +: name +: nestedFields => + val key = (catalogPart.toLowerCase(Locale.ROOT), dbPart.toLowerCase(Locale.ROOT), tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT)) - val attributes = collectMatches(name, qualified3Part.get(key)).filter { - a => (resolver(dbPart, a.qualifier.head) && resolver(tblPart, a.qualifier.last)) + val attributes = collectMatches(name, qualified4Part.get(key)).filter { a => + assert(a.qualifier.length == 3) + resolver(catalogPart, a.qualifier(0)) && resolver(dbPart, a.qualifier(1)) && + resolver(tblPart, a.qualifier(2)) } (attributes, nestedFields) case _ => (Seq.empty, Seq.empty) } + // Find matches for the given name assuming that the 1st two parts are qualifier + // (i.e. database name and table name) and the 3rd part is the actual column name. + // + // For example, consider an example where "db1" is the database name, "a" is the table name + // and "b" is the column name and "c" is the struct field name. + // If the name parts is db1.a.b.c, then it can match both + // Attribute(b, qualifier("cat", "db1, "a")) and Attribute(b, qualifier("db1, "a")), + // and List("c") will be the second element + if (matches._1.isEmpty) { + matches = nameParts match { + case dbPart +: tblPart +: name +: nestedFields => + val key = (dbPart.toLowerCase(Locale.ROOT), + tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT)) + val attributes = collectMatches(name, qualified3Part.get(key)).filter { a => + val qualifier = if (a.qualifier.length == 2) { + a.qualifier + } else { + a.qualifier.takeRight(2) + } + resolver(dbPart, qualifier.head) && resolver(tblPart, qualifier.last) + } + (attributes, nestedFields) + case _ => + (Seq.empty, Seq.empty) + } + } + // If there are no matches, then find matches for the given name assuming that // the 1st part is a qualifier (i.e. table name, alias, or subquery alias) and the // 2nd part is the actual name. This returns a tuple of @@ -219,9 +271,9 @@ package object expressions { } /** - * Match attributes for the case where at least one qualifier in `attrs` has more than 2 parts. + * Match attributes for the case where at least one qualifier in `attrs` has more than 3 parts. */ - private def matchWithThreeOrMoreQualifierParts( + private def matchWithFourOrMoreQualifierParts( nameParts: Seq[String], resolver: Resolver): (Seq[Attribute], Seq[String]) = { // Returns true if the `short` qualifier is a subset of the last elements of @@ -277,10 +329,10 @@ package object expressions { /** Perform attribute resolution given a name and a resolver. */ def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = { - val (candidates, nestedFields) = if (hasTwoOrLessQualifierParts) { - matchWithTwoOrLessQualifierParts(nameParts, resolver) + val (candidates, nestedFields) = if (hasThreeOrLessQualifierParts) { + matchWithThreeOrLessQualifierParts(nameParts, resolver) } else { - matchWithThreeOrMoreQualifierParts(nameParts, resolver) + matchWithFourOrMoreQualifierParts(nameParts, resolver) } def name = UnresolvedAttribute(nameParts).name diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala index 0d9e2f6..4d88a8d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala @@ -635,11 +635,11 @@ abstract class SessionCatalogSuite extends AnalysisTest { val view = View(desc = metadata, output = metadata.schema.toAttributes, child = CatalystSqlParser.parsePlan(metadata.viewText.get)) comparePlans(catalog.lookupRelation(TableIdentifier("view1", Some("db3"))), - SubqueryAlias("view1", "db3", view)) + SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view)) // Look up a view using current database of the session catalog. catalog.setCurrentDatabase("db3") comparePlans(catalog.lookupRelation(TableIdentifier("view1")), - SubqueryAlias("view1", "db3", view)) + SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view)) } } @@ -655,7 +655,7 @@ abstract class SessionCatalogSuite extends AnalysisTest { val view = View(desc = metadata, output = metadata.schema.toAttributes, child = CatalystSqlParser.parsePlan(metadata.viewText.get)) comparePlans(catalog.lookupRelation(TableIdentifier("view2", Some("db3"))), - SubqueryAlias("view2", "db3", view)) + SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view2"), view)) } } diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out index f34b75a..04ddfe0 100644 --- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out @@ -72,7 +72,7 @@ SELECT i1 FROM t1, mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7 -- !query @@ -81,7 +81,7 @@ SELECT t1.i1 FROM t1, mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7 -- !query @@ -90,7 +90,7 @@ SELECT mydb1.t1.i1 FROM t1, mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'mydb1.t1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7 +Reference 'mydb1.t1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7 -- !query @@ -99,7 +99,7 @@ SELECT i1 FROM t1, mydb2.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7 -- !query @@ -108,7 +108,7 @@ SELECT t1.i1 FROM t1, mydb2.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7 -- !query @@ -125,7 +125,7 @@ SELECT i1 FROM t1, mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7 -- !query @@ -134,7 +134,7 @@ SELECT t1.i1 FROM t1, mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7 -- !query @@ -143,7 +143,7 @@ SELECT i1 FROM t1, mydb2.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7 +Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7 -- !query @@ -152,7 +152,7 @@ SELECT t1.i1 FROM t1, mydb2.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7 +Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7 -- !query @@ -161,7 +161,7 @@ SELECT db1.t1.i1 FROM t1, mydb2.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -cannot resolve '`db1.t1.i1`' given input columns: [mydb2.t1.i1, mydb2.t1.i1]; line 1 pos 7 +cannot resolve '`db1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7 -- !query @@ -186,7 +186,7 @@ SELECT mydb1.t1 FROM t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7 +cannot resolve '`mydb1.t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7 -- !query @@ -204,7 +204,7 @@ SELECT t1 FROM mydb1.t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -cannot resolve '`t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7 +cannot resolve '`t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7 -- !query @@ -221,7 +221,7 @@ SELECT mydb1.t1.i1 FROM t1 struct<> -- !query output org.apache.spark.sql.AnalysisException -cannot resolve '`mydb1.t1.i1`' given input columns: [mydb2.t1.i1]; line 1 pos 7 +cannot resolve '`mydb1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out index 85ce9786..1f2bd57 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out @@ -56,7 +56,7 @@ CREATE VIEW key_dependent_view AS struct<> -- !query output org.apache.spark.sql.AnalysisException -expression 'default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.; +expression 'spark_catalog.default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.; -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out index 5332dff..20f4f6b 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out @@ -536,7 +536,7 @@ SELECT '' AS `xxx`, i, k, t struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 20 +Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 20 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out index cbf4cfa..d8d33d9 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out @@ -143,7 +143,7 @@ SELECT a FROM test_having HAVING min(a) < max(a) struct<> -- !query output org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.; +grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.; -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out index 5a52358..acce688 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out @@ -244,7 +244,7 @@ from t1 where f1 = f2 struct<> -- !query output org.apache.spark.sql.AnalysisException -cannot resolve '(PARTITION BY default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24 +cannot resolve '(PARTITION BY spark_catalog.default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24 -- !query @@ -306,7 +306,7 @@ org.apache.spark.sql.AnalysisException The query operator `Join` contains one or more unsupported expression types Aggregate, Window or Generate. -Invalid expressions: [row_number() OVER (ORDER BY default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]; +Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]; -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out index 3cc14ff..188b57f 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out @@ -536,7 +536,7 @@ SELECT udf('') AS `xxx`, udf(i) AS i, udf(k), udf(t) AS t struct<> -- !query output org.apache.spark.sql.AnalysisException -Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 29 +Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 29 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out index bb108a2..50b6e60 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out @@ -143,7 +143,7 @@ SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a)) struct<> -- !query output org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.; +grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.; -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 563b4d1..81dfa798 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -2797,7 +2797,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark sql("SELECT * FROM t, S WHERE c = C") }.message assert( - m.contains("cannot resolve '(default.t.`c` = default.S.`C`)' due to data type mismatch")) + m.contains( + "cannot resolve '(spark_catalog.default.t.`c` = spark_catalog.default.S.`C`)' " + + "due to data type mismatch")) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 1fc0bb1..ba4200d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -758,14 +758,10 @@ class DataSourceV2SQLSuite checkAnswer(sql("select i from t"), Row(1)) checkAnswer(sql("select t.i from t"), Row(1)) checkAnswer(sql("select default.t.i from t"), Row(1)) + checkAnswer(sql("select spark_catalog.default.t.i from t"), Row(1)) checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1)) checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1)) - - // catalog name cannot be used for tables in the session catalog. - val ex = intercept[AnalysisException] { - sql(s"select spark_catalog.default.t.i from spark_catalog.default.t") - } - assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`")) + checkAnswer(sql("select spark_catalog.default.t.i from spark_catalog.default.t"), Row(1)) } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala index b8ef44b..9cd56f1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala @@ -62,7 +62,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils { spark.sql("create view vw1 as select 1 as id") val plan = spark.sql("select id from vw1").queryExecution.analyzed val aliases = plan.collect { - case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("default")), _) => x + case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("spark_catalog", "default")), _) => x } assert(aliases.size == 1) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org