This is an automated email from the ASF dual-hosted git repository. ptoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 6a9d35f766d [SPARK-45354][SQL] Resolve functions bottom-up 6a9d35f766d is described below commit 6a9d35f766dcc323a2f20326fbfa4675fe51b8d7 Author: Peter Toth <peter.t...@gmail.com> AuthorDate: Wed Sep 27 21:40:16 2023 +0200 [SPARK-45354][SQL] Resolve functions bottom-up ### What changes were proposed in this pull request? This PR proposes bottum-up resolution in `ResolveFunctions`, which is much faster (requires less number of resolution rounds) if we have deeply nested `UnresolvedFunctions`. These structures are more likely to occur after https://github.com/apache/spark/pull/42864. ### Why are the changes needed? Performance optimization. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing UTs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43146 from peter-toth/SPARK-45354-resolve-functions-bottom-up. Authored-by: Peter Toth <peter.t...@gmail.com> Signed-off-by: Peter Toth <peter.t...@gmail.com> --- .../scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +- .../test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out | 6 +++--- sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index aac85e19721..67a958d73f7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2207,7 +2207,7 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor Project(aliases, u.child) case q: LogicalPlan => - q.transformExpressionsWithPruning( + q.transformExpressionsUpWithPruning( _.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR), ruleId) { case u @ UnresolvedFunction(nameParts, arguments, _, _, _) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out index 27b9786160c..eca4a8ba02f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out @@ -72,9 +72,9 @@ org.apache.spark.sql.AnalysisException "queryContext" : [ { "objectType" : "", "objectName" : "", - "startIndex" : 95, - "stopIndex" : 117, - "fragment" : "default.udaf1(int_col1)" + "startIndex" : 8, + "stopIndex" : 35, + "fragment" : "default.udaf1(udf(int_col1))" } ] } diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out index 228a31ba257..f33e4b40395 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out @@ -73,9 +73,9 @@ org.apache.spark.sql.AnalysisException "queryContext" : [ { "objectType" : "", "objectName" : "", - "startIndex" : 95, - "stopIndex" : 117, - "fragment" : "default.udaf1(int_col1)" + "startIndex" : 8, + "stopIndex" : 35, + "fragment" : "default.udaf1(udf(int_col1))" } ] } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org