This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 9934b56dbb2 [SPARK-41350][3.3][SQL][FOLLOWUP] Allow simple name access
of join hidden columns after alias
9934b56dbb2 is described below
commit 9934b56dbb23debfdc02340b493263a665189c0c
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Thu Dec 22 11:26:19 2022 +0800
[SPARK-41350][3.3][SQL][FOLLOWUP] Allow simple name access of join hidden
columns after alias
backport https://github.com/apache/spark/pull/39077 to 3.3
Closes #39121 from cloud-fan/backport.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../catalyst/expressions/namedExpressions.scala | 7 +++-
.../resources/sql-tests/inputs/natural-join.sql | 2 +
.../test/resources/sql-tests/inputs/using-join.sql | 8 ++++
.../sql-tests/results/natural-join.sql.out | 10 +++++
.../resources/sql-tests/results/using-join.sql.out | 44 ++++++++++++++++++++++
5 files changed, 69 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index a399121298b..cb2225bcfc7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
import org.apache.spark.sql.catalyst.trees.TreePattern
import org.apache.spark.sql.catalyst.trees.TreePattern._
-import org.apache.spark.sql.catalyst.util.{quoteIfNeeded,
METADATA_COL_ATTR_KEY}
+import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.types._
import org.apache.spark.util.collection.BitSet
@@ -191,7 +191,10 @@ case class Alias(child: Expression, name: String)(
override def toAttribute: Attribute = {
if (resolved) {
- AttributeReference(name, child.dataType, child.nullable,
metadata)(exprId, qualifier)
+ val a = AttributeReference(name, child.dataType, child.nullable,
metadata)(exprId, qualifier)
+ // Alias has its own qualifier. It doesn't make sense to still restrict
the hidden columns
+ // of natural/using join to be accessed by qualified name only.
+ if (a.qualifiedAccessOnly) a.markAsAllowAnyAccess() else a
} else {
UnresolvedAttribute.quoted(name)
}
diff --git a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
index 9c9ce6c37ba..5815eee2d9e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql
@@ -52,6 +52,8 @@ SELECT nt1.k, nt2.k FROM nt1 natural join nt2;
SELECT k FROM (SELECT nt2.k FROM nt1 natural join nt2);
+SELECT nt2.k AS key FROM nt1 natural join nt2 ORDER BY key;
+
SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one";
SELECT * FROM (SELECT * FROM nt1 natural join nt2);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
index 414221e5b71..0c06023b14d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/using-join.sql
@@ -21,6 +21,8 @@ SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k);
SELECT k FROM (SELECT nt2.k FROM nt1 left outer join nt2 using (k));
+SELECT nt2.k AS key FROM nt1 left outer join nt2 using (k) ORDER BY key;
+
SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k) ORDER BY nt2.k;
SELECT k, nt1.k FROM nt1 left outer join nt2 using (k);
@@ -47,6 +49,8 @@ SELECT nt1.k, nt2.k FROM nt1 right outer join nt2 using (k);
SELECT k FROM (SELECT nt1.k FROM nt1 right outer join nt2 using (k));
+SELECT nt1.k AS key FROM nt1 right outer join nt2 using (k) ORDER BY key;
+
SELECT k, nt1.k FROM nt1 right outer join nt2 using (k);
SELECT k, nt2.k FROM nt1 right outer join nt2 using (k);
@@ -61,6 +65,8 @@ SELECT nt1.k, nt2.k FROM nt1 full outer join nt2 using (k);
SELECT k FROM (SELECT nt2.k FROM nt1 full outer join nt2 using (k));
+SELECT nt2.k AS key FROM nt1 full outer join nt2 using (k) ORDER BY key;
+
SELECT k, nt1.k FROM nt1 full outer join nt2 using (k);
SELECT k, nt2.k FROM nt1 full outer join nt2 using (k);
@@ -75,6 +81,8 @@ SELECT nt1.k, nt2.k FROM nt1 inner join nt2 using (k);
SELECT k FROM (SELECT nt2.k FROM nt1 inner join nt2 using (k));
+SELECT nt2.k AS key FROM nt1 inner join nt2 using (k) ORDER BY key;
+
SELECT k, nt1.k FROM nt1 inner join nt2 using (k);
SELECT k, nt2.k FROM nt1 inner join nt2 using (k);
diff --git a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
index 4bba2738594..dc6acc4ef91 100644
--- a/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/natural-join.sql.out
@@ -197,6 +197,16 @@ one
two
+-- !query
+SELECT nt2.k AS key FROM nt1 natural join nt2 ORDER BY key
+-- !query schema
+struct<key:string>
+-- !query output
+one
+one
+two
+
+
-- !query
SELECT nt1.k, nt2.k FROM nt1 natural join nt2 where k = "one"
-- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
index e0977435546..f1b0c8f44fe 100644
--- a/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/using-join.sql.out
@@ -82,6 +82,17 @@ one
two
+-- !query
+SELECT nt2.k AS key FROM nt1 left outer join nt2 using (k) ORDER BY key
+-- !query schema
+struct<key:string>
+-- !query output
+NULL
+one
+one
+two
+
+
-- !query
SELECT nt1.k, nt2.k FROM nt1 left outer join nt2 using (k) ORDER BY nt2.k
-- !query schema
@@ -215,6 +226,17 @@ one
two
+-- !query
+SELECT nt1.k AS key FROM nt1 right outer join nt2 using (k) ORDER BY key
+-- !query schema
+struct<key:string>
+-- !query output
+NULL
+one
+one
+two
+
+
-- !query
SELECT k, nt1.k FROM nt1 right outer join nt2 using (k)
-- !query schema
@@ -297,6 +319,18 @@ one
two
+-- !query
+SELECT nt2.k AS key FROM nt1 full outer join nt2 using (k) ORDER BY key
+-- !query schema
+struct<key:string>
+-- !query output
+NULL
+four
+one
+one
+two
+
+
-- !query
SELECT k, nt1.k FROM nt1 full outer join nt2 using (k)
-- !query schema
@@ -373,6 +407,16 @@ one
two
+-- !query
+SELECT nt2.k AS key FROM nt1 inner join nt2 using (k) ORDER BY key
+-- !query schema
+struct<key:string>
+-- !query output
+one
+one
+two
+
+
-- !query
SELECT k, nt1.k FROM nt1 inner join nt2 using (k)
-- !query schema
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
