This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e0b3f2abe8ba [SPARK-49235][SQL] Refactor ResolveInlineTables rule so
it doesn't traverse the whole tree
e0b3f2abe8ba is described below
commit e0b3f2abe8bab7c2768901022f5d112fbf941cf1
Author: Mihailo Aleksic <mihailo.alek...@databricks.com>
AuthorDate: Wed Aug 14 22:06:09 2024 +0200
[SPARK-49235][SQL] Refactor ResolveInlineTables rule so it doesn't traverse
the whole tree
### What changes were proposed in this pull request?
Refactoring `EvalInlineTables` in a way so we can use its body to resolve
inline tables using `ResolveInlineTables` rule without traversing the whole
tree.
### Why are the changes needed?
In the description of `EvalInlineTables` you can see that it is an
optimizer rule which should run at the very end of the analysis phase.
Nevertheless it is invoked while resolving inline tables in the analysis phase.
We discovered that it can be done without traversing the whole tree so we
refactored `EvalInlineTables` in a way that we can use it's body in
`ResolveInlineTables` which improves readability of the code and its
performance a bit.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47760 from mihailoale-db/mihailoale-db/refactor_evalinlinetables.
Authored-by: Mihailo Aleksic <mihailo.alek...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../sql/catalyst/optimizer/finishAnalysis.scala | 33 ++++++++++++----------
.../util/EvaluateUnresolvedInlineTable.scala | 2 +-
2 files changed, 19 insertions(+), 16 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 48753fbfe326..55f222d88084 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -84,23 +84,26 @@ object RewriteNonCorrelatedExists extends Rule[LogicalPlan]
{
object EvalInlineTables extends Rule[LogicalPlan] with CastSupport {
override def apply(plan: LogicalPlan): LogicalPlan = {
plan.transformDownWithSubqueriesAndPruning(_.containsPattern(INLINE_TABLE_EVAL))
{
- case table: ResolvedInlineTable =>
- val newRows: Seq[InternalRow] =
- table.rows.map { row => InternalRow.fromSeq(row.map { e =>
- try {
- prepareForEval(e).eval()
- } catch {
- case NonFatal(ex) =>
- table.failAnalysis(
- errorClass =
"INVALID_INLINE_TABLE.FAILED_SQL_EXPRESSION_EVALUATION",
- messageParameters = Map("sqlExpr" -> toSQLExpr(e)),
- cause = ex)
- }})
- }
-
- LocalRelation(table.output, newRows)
+ case table: ResolvedInlineTable => eval(table)
}
}
+
+ def eval(table: ResolvedInlineTable): LocalRelation = {
+ val newRows: Seq[InternalRow] =
+ table.rows.map { row => InternalRow.fromSeq(row.map { e =>
+ try {
+ prepareForEval(e).eval()
+ } catch {
+ case NonFatal(ex) =>
+ table.failAnalysis(
+ errorClass =
"INVALID_INLINE_TABLE.FAILED_SQL_EXPRESSION_EVALUATION",
+ messageParameters = Map("sqlExpr" -> toSQLExpr(e)),
+ cause = ex)
+ }})
+ }
+
+ LocalRelation(table.output, newRows)
+ }
}
/**
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/EvaluateUnresolvedInlineTable.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/EvaluateUnresolvedInlineTable.scala
index 63af2b3f5ffc..a55f70c238a8 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/EvaluateUnresolvedInlineTable.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/EvaluateUnresolvedInlineTable.scala
@@ -62,7 +62,7 @@ object EvaluateUnresolvedInlineTable extends SQLConfHelper
*/
private def earlyEvalIfPossible(table: ResolvedInlineTable): LogicalPlan = {
val earlyEvalPossible =
table.rows.flatten.forall(!_.containsPattern(CURRENT_LIKE))
- if (earlyEvalPossible) EvalInlineTables(table) else table
+ if (earlyEvalPossible) EvalInlineTables.eval(table) else table
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
