This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new 9934b56dbb2 [SPARK-41350][3.3][SQL][FOLLOWUP] Allow simple name access of join hidden columns after alias 9934b56dbb2 is described below commit 9934b56dbb23debfdc02340b493263a665189c0c Author: Wenchen Fan <wenc...@databricks.com> AuthorDate: Thu Dec 22 11:26:19 2022 +0800 [SPARK-41350][3.3][SQL][FOLLOWUP] Allow simple name access of join hidden columns after alias backport https://github.com/apache/spark/pull/39077 to 3.3 Closes #39121 from cloud-fan/backport. Authored-by: Wenchen Fan <wenc...@databricks.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../catalyst/expressions/namedExpressions.scala | 7 +++- .../resources/sql-tests/inputs/natural-join.sql | 2 + .../test/resources/sql-tests/inputs/using-join.sql | 8 ++++ .../sql-tests/results/natural-join.sql.out | 10 +++++ .../resources/sql-tests/results/using-join.sql.out | 44 ++++++++++++++++++++++ 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index a399121298b..cb2225bcfc7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark import org.apache.spark.sql.catalyst.trees.TreePattern import org.apache.spark.sql.catalyst.trees.TreePattern._ -import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, METADATA_COL_ATTR_KEY} +import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.types._ import org.apache.spark.util.collection.BitSet @@ -191,7 +191,10 @@ case class Alias(child: Expression, name: String)( override def toAttribute: Attribute = { if (resolved) { - AttributeReference(name, child.dataType, child.nullable, metadata)(exprId, qualifier) + val a = AttributeReference(name, child.dataType, child.nullable, metadata)(exprId, qualifier) + // Alias has its own qualifier. It doesn't make sense to still restrict the hidden columns + // of natural/using join to be accessed by qualified name only. + if (a.qualifiedAccessOnly) a.markAsAllowAnyAccess() else a } else { UnresolvedAttribute.quoted(name) } diff --git a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql index 9c9ce6c37ba..5815eee2d9e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql @@ -52,6 +52,8 @@ SELECT nt1.k, nt2.k FROM nt1 natural join nt2; SELECT k FROM (SELECT nt2.k FROM nt1 natural join nt2); +SELECT nt2.k AS key FROM nt1 natural join nt2 ORDER BY key; + SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one"; SELECT * FROM (SELECT * FROM nt1 natural join nt2); diff --git a/sql/core/src/test/resources/sql-tests/inputs/using-join.sql b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql index 414221e5b71..0c06023b14d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/using-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql @@ -21,6 +21,8 @@ SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k); SELECT k FROM (SELECT nt2.k FROM nt1 left outer join nt2 using (k)); +SELECT nt2.k AS key FROM nt1 left outer join nt2 using (k) ORDER BY key; + SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k) ORDER BY nt2.k; SELECT k, nt1.k FROM nt1 left outer join nt2 using (k); @@ -47,6 +49,8 @@ SELECT nt1.k, nt2.k FROM nt1 right outer join nt2 using (k); SELECT k FROM (SELECT nt1.k FROM nt1 right outer join nt2 using (k)); +SELECT nt1.k AS key FROM nt1 right outer join nt2 using (k) ORDER BY key; + SELECT k, nt1.k FROM nt1 right outer join nt2 using (k); SELECT k, nt2.k FROM nt1 right outer join nt2 using (k); @@ -61,6 +65,8 @@ SELECT nt1.k, nt2.k FROM nt1 full outer join nt2 using (k); SELECT k FROM (SELECT nt2.k FROM nt1 full outer join nt2 using (k)); +SELECT nt2.k AS key FROM nt1 full outer join nt2 using (k) ORDER BY key; + SELECT k, nt1.k FROM nt1 full outer join nt2 using (k); SELECT k, nt2.k FROM nt1 full outer join nt2 using (k); @@ -75,6 +81,8 @@ SELECT nt1.k, nt2.k FROM nt1 inner join nt2 using (k); SELECT k FROM (SELECT nt2.k FROM nt1 inner join nt2 using (k)); +SELECT nt2.k AS key FROM nt1 inner join nt2 using (k) ORDER BY key; + SELECT k, nt1.k FROM nt1 inner join nt2 using (k); SELECT k, nt2.k FROM nt1 inner join nt2 using (k); diff --git a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out index 4bba2738594..dc6acc4ef91 100644 --- a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out @@ -197,6 +197,16 @@ one two +-- !query +SELECT nt2.k AS key FROM nt1 natural join nt2 ORDER BY key +-- !query schema +struct<key:string> +-- !query output +one +one +two + + -- !query SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one" -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/using-join.sql.out b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out index e0977435546..f1b0c8f44fe 100644 --- a/sql/core/src/test/resources/sql-tests/results/using-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out @@ -82,6 +82,17 @@ one two +-- !query +SELECT nt2.k AS key FROM nt1 left outer join nt2 using (k) ORDER BY key +-- !query schema +struct<key:string> +-- !query output +NULL +one +one +two + + -- !query SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k) ORDER BY nt2.k -- !query schema @@ -215,6 +226,17 @@ one two +-- !query +SELECT nt1.k AS key FROM nt1 right outer join nt2 using (k) ORDER BY key +-- !query schema +struct<key:string> +-- !query output +NULL +one +one +two + + -- !query SELECT k, nt1.k FROM nt1 right outer join nt2 using (k) -- !query schema @@ -297,6 +319,18 @@ one two +-- !query +SELECT nt2.k AS key FROM nt1 full outer join nt2 using (k) ORDER BY key +-- !query schema +struct<key:string> +-- !query output +NULL +four +one +one +two + + -- !query SELECT k, nt1.k FROM nt1 full outer join nt2 using (k) -- !query schema @@ -373,6 +407,16 @@ one two +-- !query +SELECT nt2.k AS key FROM nt1 inner join nt2 using (k) ORDER BY key +-- !query schema +struct<key:string> +-- !query output +one +one +two + + -- !query SELECT k, nt1.k FROM nt1 inner join nt2 using (k) -- !query schema --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org