This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 714884cb115f [SPARK-51625][SQL] Command in CTE relations should
trigger inline
714884cb115f is described below
commit 714884cb115f2520a8bf3a44d7529c21abbabf40
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Thu Mar 27 15:46:36 2025 +0800
[SPARK-51625][SQL] Command in CTE relations should trigger inline
### What changes were proposed in this pull request?
This PR fixes a small issue in the rule `CTESubstitution`. The intention is
to fall back to CTE inline if there are commands in the plan. However, the code
simply does `plan.collect` which misses the fact that `UnresolvedWith` does not
include the CTE relations in its children.
To fix it, we should do a manually recursion to handle the CTE relations.
### Why are the changes needed?
A potential bug as the code does not work as expected.
### Does this PR introduce _any_ user-facing change?
No, there is no user-facing API that can construct such a CTE plan.
### How was this patch tested?
a new test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #50425 from cloud-fan/cte.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../apache/spark/sql/catalyst/analysis/CTESubstitution.scala | 10 ++++++++--
.../src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala | 12 ++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index ef13bc191db5..4bc3300b7574 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -56,9 +56,15 @@ object CTESubstitution extends Rule[LogicalPlan] {
return plan
}
- val commands = plan.collect {
- case c @ (_: Command | _: ParsedStatement | _: InsertIntoDir) => c
+ def collectCommands(p: LogicalPlan): Seq[LogicalPlan] = p match {
+ case c @ (_: Command | _: ParsedStatement | _: InsertIntoDir) => Seq(c)
+ case u: UnresolvedWith =>
+ collectCommands(u.child) ++ u.cteRelations.flatMap {
+ case (_, relation) => collectCommands(relation)
+ }
+ case p => p.children.flatMap(collectCommands)
}
+ val commands = collectCommands(plan)
val forceInline = if (commands.length == 1) {
if (conf.getConf(SQLConf.LEGACY_INLINE_CTE_IN_COMMANDS)) {
// The legacy behavior always inlines the CTE relations for queries in
commands.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
index 9bddaf3baab0..80450b79a9a7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql
+import org.apache.spark.sql.catalyst.analysis.{CurrentNamespace,
UnresolvedRelation}
import org.apache.spark.sql.catalyst.expressions.{Alias, And, GreaterThan,
LessThan, Literal, Or, Rand}
import org.apache.spark.sql.catalyst.optimizer.InlineCTE
import org.apache.spark.sql.catalyst.plans.logical._
@@ -834,6 +835,17 @@ abstract class CTEInlineSuiteBase
}
}
}
+
+ test("SPARK-51625: command in CTE relations should trigger inline") {
+ val plan = UnresolvedWith(
+ child = UnresolvedRelation(Seq("t")),
+ cteRelations = Seq("t" -> SubqueryAlias("t",
ShowTables(CurrentNamespace, pattern = None)))
+ )
+ assert(!spark.sessionState.analyzer.execute(plan).exists {
+ case _: WithCTE => true
+ case _ => false
+ })
+ }
}
class CTEInlineSuiteAEOff extends CTEInlineSuiteBase with
DisableAdaptiveExecutionSuite
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
