[GitHub] [spark] wangyum commented on a diff in pull request #36080: [SPARK-38797][SQL] Runtime Filter supports pruning side has window

[GitBox]

wangyum commented on code in PR #36080:
URL: https://github.com/apache/spark/pull/36080#discussion_r843661577


##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/InjectRuntimeFilter.scala:
##########
@@ -132,28 +131,21 @@ object InjectRuntimeFilter extends Rule[LogicalPlan] with 
PredicateHelper with J
       REGEXP_EXTRACT_FAMILY, REGEXP_REPLACE)
   }
 
-  private def canFilterLeft(joinType: JoinType): Boolean = joinType match {
-    case Inner | RightOuter => true
-    case _ => false
-  }
-
-  private def canFilterRight(joinType: JoinType): Boolean = joinType match {
-    case Inner | LeftOuter => true
-    case _ => false
-  }
-
   private def isProbablyShuffleJoin(left: LogicalPlan,
       right: LogicalPlan, hint: JoinHint): Boolean = {
     !hintToBroadcastLeft(hint) && !hintToBroadcastRight(hint) &&
       !canBroadcastBySize(left, conf) && !canBroadcastBySize(right, conf)
   }
 
-  private def probablyHasShuffle(plan: LogicalPlan): Boolean = {
-    plan.collectFirst {
-      case j@Join(left, right, _, _, hint)
-        if isProbablyShuffleJoin(left, right, hint) => j
-      case a: Aggregate => a
-    }.nonEmpty
+  private def probablyHasShuffle(exp: Expression, plan: LogicalPlan): Boolean 
= {
+    plan.exists {
+      case Join(left, right, _, _, hint)
+        if isProbablyShuffleJoin(left, right, hint) => true
+      case _: Aggregate => true
+      case w: Window
+        if 
exp.references.subsetOf(AttributeSet(w.partitionSpec.flatMap(_.references))) => 
true

Review Comment:
   +1



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org