minyyy commented on code in PR #35864:
URL: https://github.com/apache/spark/pull/35864#discussion_r842096650
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala:
##########
@@ -312,6 +312,32 @@ object NestedColumnAliasing {
}
}
+object GeneratorUnrequiredChildrenPruning {
+ def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
+ case p@Project(_, g: Generate) =>
+ val requiredAttrs = p.references ++ g.generator.references
+ var pruned = false
+ val newChild = if (!g.child.outputSet.subsetOf(requiredAttrs)) {
+ pruned = true
+ Project(g.child.output.filter(requiredAttrs.contains), g.child)
+ } else {
+ g.child
+ }
+ val unrequired = g.generator.references -- p.references
+ val unrequiredIndices = newChild.output.zipWithIndex.filter(t =>
unrequired.contains(t._1))
+ .map(_._2)
+ if (unrequiredIndices.toSet != g.unrequiredChildIndex.toSet) {
+ pruned = true
+ }
+ if (pruned) {
+ Some(p.copy(child = g.copy(child = newChild, unrequiredChildIndex =
unrequiredIndices)))
+ } else {
+ None
+ }
Review Comment:
The only logic change is that I changed
```val requiredAttrs = p.references -- g.producedAttributes ++
g.generator.references```
to
```val requiredAttrs = p.references ++ g.generator.references```
The other changes is due to `prunedChild` is defined in the `ColumnPruning`
class so I just inlined its logic.
For the change to `requiredAttrs`, the reason is that I think the previous
logic is incorrect, I don't think `g.producedAttributes` should be removed from
the `requiredAttrs` list.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
