ConeyLiu commented on code in PR #9233:
URL: https://github.com/apache/iceberg/pull/9233#discussion_r1419107868
##########
spark/v3.4/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceStaticInvoke.scala:
##########
@@ -39,22 +41,31 @@ import org.apache.spark.sql.types.StructType
*/
object ReplaceStaticInvoke extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan =
- plan.transformWithPruning (_.containsAllPatterns(BINARY_COMPARISON,
FILTER)) {
- case filter @ Filter(condition, _) =>
- val newCondition =
condition.transformWithPruning(_.containsPattern(BINARY_COMPARISON)) {
- case c @ BinaryComparison(left: StaticInvoke, right) if
canReplace(left) && right.foldable =>
- c.withNewChildren(Seq(replaceStaticInvoke(left), right))
+ private val rule:PartialFunction[Expression, Expression] = {
+ case c@BinaryComparison(left: StaticInvoke, right) if canReplace(left) &&
right.foldable =>
+ c.withNewChildren(Seq(replaceStaticInvoke(left), right))
- case c @ BinaryComparison(left, right: StaticInvoke) if
canReplace(right) && left.foldable =>
- c.withNewChildren(Seq(left, replaceStaticInvoke(right)))
- }
+ case c@BinaryComparison(left, right: StaticInvoke) if canReplace(right) &&
left.foldable =>
+ c.withNewChildren(Seq(left, replaceStaticInvoke(right)))
+ }
+ override def apply(plan: LogicalPlan): LogicalPlan = {
+ plan.transformWithPruning(_.containsAnyPattern(FILTER, JOIN)) {
+ case filter @ Filter(condition, _) =>
+ val newCondition =
condition.transformWithPruning(_.containsPattern(BINARY_COMPARISON))(rule)
if (newCondition fastEquals condition) {
filter
} else {
filter.copy(condition = newCondition)
}
+ case j @ Join(_, _, _, Some(condition), _) =>
Review Comment:
Here the join condition can be pushed to the leaf node by the Spark
optimizer, right? I think this can not cover the COW/MOR cases. COW/MOR needs
to do some special handling here. I plan to do it, however, I've been quite
busy lately.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
