This is an automated email from the ASF dual-hosted git repository.
ptoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6a9d35f766d [SPARK-45354][SQL] Resolve functions bottom-up
6a9d35f766d is described below
commit 6a9d35f766dcc323a2f20326fbfa4675fe51b8d7
Author: Peter Toth <peter.t...@gmail.com>
AuthorDate: Wed Sep 27 21:40:16 2023 +0200
[SPARK-45354][SQL] Resolve functions bottom-up
### What changes were proposed in this pull request?
This PR proposes bottum-up resolution in `ResolveFunctions`, which is much
faster (requires less number of resolution rounds) if we have deeply nested
`UnresolvedFunctions`. These structures are more likely to occur after
https://github.com/apache/spark/pull/42864.
### Why are the changes needed?
Performance optimization.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing UTs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43146 from peter-toth/SPARK-45354-resolve-functions-bottom-up.
Authored-by: Peter Toth <peter.t...@gmail.com>
Signed-off-by: Peter Toth <peter.t...@gmail.com>
---
.../scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out | 6 +++---
sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out | 6 +++---
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index aac85e19721..67a958d73f7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2207,7 +2207,7 @@ class Analyzer(override val catalogManager:
CatalogManager) extends RuleExecutor
Project(aliases, u.child)
case q: LogicalPlan =>
- q.transformExpressionsWithPruning(
+ q.transformExpressionsUpWithPruning(
_.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR),
ruleId) {
case u @ UnresolvedFunction(nameParts, arguments, _, _, _)
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out
index 27b9786160c..eca4a8ba02f 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/udf-udaf.sql.out
@@ -72,9 +72,9 @@ org.apache.spark.sql.AnalysisException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 95,
- "stopIndex" : 117,
- "fragment" : "default.udaf1(int_col1)"
+ "startIndex" : 8,
+ "stopIndex" : 35,
+ "fragment" : "default.udaf1(udf(int_col1))"
} ]
}
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out
b/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out
index 228a31ba257..f33e4b40395 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-udaf.sql.out
@@ -73,9 +73,9 @@ org.apache.spark.sql.AnalysisException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 95,
- "stopIndex" : 117,
- "fragment" : "default.udaf1(int_col1)"
+ "startIndex" : 8,
+ "stopIndex" : 35,
+ "fragment" : "default.udaf1(udf(int_col1))"
} ]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
