This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push: new 83f8ddd1ac2 [SPARK-42416][SQL] Dateset operations should not resolve the analyzed logical plan again 83f8ddd1ac2 is described below commit 83f8ddd1ac285550fdd429e96a37f101ef23c79c Author: Gengliang Wang <gengli...@apache.org> AuthorDate: Mon Feb 13 10:57:47 2023 -0800 [SPARK-42416][SQL] Dateset operations should not resolve the analyzed logical plan again ### What changes were proposed in this pull request? For the following query ``` sql( """ |CREATE TABLE app ( | uid STRING, | st TIMESTAMP, | ds INT |) USING parquet PARTITIONED BY (ds); |""".stripMargin) sql( """ |create or replace temporary view view1 as WITH new_app AS ( | SELECT a.* FROM app a) |SELECT | uid, | 20230208 AS ds | FROM | new_app | GROUP BY | 1, | 2 |""".stripMargin) val df = sql("select uid from view1") df.show() ``` Spark will throw the following error ``` [GROUP_BY_POS_OUT_OF_RANGE] GROUP BY position 20230208 is not in select list (valid range is [1, 2]).; line 9 pos 4 ``` This is because the logical plan in `df` is not set as analyzed after changes in https://github.com/apache/spark/commit/6adda258e5155761a861a96af4f5410b8a7f304d#diff-583171e935b2dc349378063a5841c5b98b30a2d57ac3743a9eccfe7bffcb8f2aR126, which sets the result of `inlineCTE(plan)` as analyzed instead. Resolving ordinals is not idempotent. In `df`, the `GROUP BY 1, 2` is resolved as `GROUP BY uid, 20230208`. `Dateset.show()` will resolve the logical plan in `df` again and treat the 20230208 in `GROUP BY uid, 20230208` as ordinals again. Thus the error happens ### Why are the changes needed? Bug fix ### Does this PR introduce _any_ user-facing change? No, the regression is not released yet. ### How was this patch tested? New UT Closes #39988 from gengliangwang/group_by_error. Authored-by: Gengliang Wang <gengli...@apache.org> Signed-off-by: Gengliang Wang <gengli...@apache.org> (cherry picked from commit 5267ad82d992ad80dccda8f59d99d84c3e85a32c) Signed-off-by: Gengliang Wang <gengli...@apache.org> --- .../sql/catalyst/analysis/CheckAnalysis.scala | 3 +- .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 34 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index e95c21ad985..77948735dbe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -142,6 +142,7 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB } // Inline all CTEs in the plan to help check query plan structures in subqueries. checkAnalysis0(inlineCTE(plan)) + plan.setAnalyzed() } def checkAnalysis0(plan: LogicalPlan): Unit = { @@ -775,8 +776,6 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB summary = o.origin.context.summary) case _ => } - - plan.setAnalyzed() } private def getAllExpressions(plan: LogicalPlan): Seq[Expression] = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index f0566619e74..3350858b7e2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -4546,6 +4546,40 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark Seq(Row(2), Row(1))) } + test("SPARK-42416: Dateset operations should not resolve the analyzed logical plan again") { + withTable("app") { + withView("view1") { + sql( + """ + |CREATE TABLE app ( + | uid STRING, + | st TIMESTAMP, + | ds INT + |) USING parquet PARTITIONED BY (ds); + |""".stripMargin) + + sql( + """ + |create or replace temporary view view1 as WITH new_app AS ( + | SELECT a.* FROM app a) + |SELECT + | uid, + | 20230208 AS ds + | FROM + | new_app + | GROUP BY + | 1, + | 2 + |""".stripMargin) + val df = sql("select uid from view1") + // If the logical plan in `df` is analyzed again, the 'group by 20230208' will be + // treated as ordinal again and there will be an error about GROUP BY position 20230208 + // being out of range. + df.show() + } + } + } + test("SPARK-39548: CreateView will make queries go into inline CTE code path thus" + "trigger a mis-clarified `window definition not found` issue") { sql( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org