This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3e27543128c8 [SPARK-48419][SQL] Foldable propagation replace foldable
column shoul…
3e27543128c8 is described below
commit 3e27543128c84bb4b6642589bb1c6da21c38b957
Author: KnightChess <981159...@qq.com>
AuthorDate: Thu May 30 17:53:38 2024 -0700
[SPARK-48419][SQL] Foldable propagation replace foldable column shoul…
…d use origin column name
### What changes were proposed in this pull request?
fix optimizer rule `FoldablePropagation` will change column name, use
origin name.
### Why are the changes needed?
fix bug
### Does this PR introduce _any_ user-facing change?
`before fix`
befor optimizer:
```shell
'Project ['x, 'y, 'z]
+- 'Project ['a AS x, str AS Y, 'b AS z]
+- LocalRelation <empty> , [a, b]
```
after optimizer:
```shell
Project [x, str AS Y, z]
+- Project [a#0 AS x#112, str AS Y#113, b#1 AS z#114]
+- LocalRelation <empty>, [a, b]
```
column name `y` will be replace to 'Y', it change plan schame
`after fix`
the query plan schema is still y
```shell
Project [x, str AS y, z]
+- Project [a#0 AS x#112, str AS Y#113, b#1 AS z#114]
+- LocalRelation <empty>, [a, b]
```
### How was this patch tested?
Added UT
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #46742 from KnightChess/fix-foldable-propagation.
Authored-by: KnightChess <981159...@qq.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../org/apache/spark/sql/catalyst/optimizer/expressions.scala | 2 +-
.../sql/catalyst/optimizer/FoldablePropagationSuite.scala | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 32700f176f25..2c55e4c8fd37 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -1023,7 +1023,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
plan
} else {
plan transformExpressions {
- case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+ case a: AttributeReference if foldableMap.contains(a) =>
foldableMap(a).withName(a.name)
}
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
index 767ef38ea7f7..5866f29e4e86 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
@@ -214,4 +214,15 @@ class FoldablePropagationSuite extends PlanTest {
val expected = testRelation.select(foldableAttr,
$"a").rebalance(foldableAttr, $"a").analyze
comparePlans(optimized, expected)
}
+
+ test("SPARK-48419: Foldable propagation replace foldable column should use
origin column name") {
+ val query = testRelation
+ .select($"a".as("x"), "str".as("Y"), $"b".as("z"))
+ .select($"x", $"y", $"z")
+ val optimized = Optimize.execute(query.analyze)
+ val correctAnswer = testRelation
+ .select($"a".as("x"), "str".as("Y"), $"b".as("z"))
+ .select($"x", "str".as("y"), $"z").analyze
+ comparePlans(optimized, correctAnswer)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
