eejbyfeldt commented on code in PR #56077:
URL: https://github.com/apache/spark/pull/56077#discussion_r3301486256
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteSubquerySuite.scala:
##########
@@ -96,4 +96,33 @@ class RewriteSubquerySuite extends PlanTest {
.select($"exists".as("(sum(col2) IN (listquery()))")).analyze
comparePlans(optimized, correctAnswer)
}
+
+ test("SPARK-57005: No None.get when correlated predicates are eliminated") {
+ // When BooleanSimplification in PullupCorrelatedPredicates eliminates all
correlated
+ // predicates (e.g., FALSE AND correlated_pred -> FALSE), the Exists node
ends up with
+ // outerAttrs non-empty but joinCond empty. RewritePredicateSubquery must
handle this.
+ object OptimizeWithPullup extends RuleExecutor[LogicalPlan] {
+ val batches =
+ Batch("Pullup Correlated Expressions", Once,
+ PullupCorrelatedPredicates) ::
+ Batch("Rewrite Subquery", FixedPoint(1),
+ RewritePredicateSubquery,
+ ColumnPruning,
+ CollapseProject,
+ RemoveNoopOperators) :: Nil
+ }
+
+ val outer = LocalRelation($"a".int, $"b".int)
+ val inner = LocalRelation($"x".int, $"y".int)
+
+ // NOT EXISTS with FALSE AND correlated_pred - should not throw None.get
+ val notExistsQuery = outer.where(
+ Not(Exists(inner.where(Literal.FalseLiteral && $"a" ===
$"x")))).select($"a")
+ OptimizeWithPullup.execute(notExistsQuery.analyze)
+
+ // EXISTS with FALSE AND correlated_pred - should not throw None.get
+ val existsQuery = outer.where(
+ Exists(inner.where(Literal.FalseLiteral && $"a" === $"x"))).select($"a")
+ OptimizeWithPullup.execute(existsQuery.analyze)
Review Comment:
The pullup rule is not needed here to reproduce the crash? What the
motivation for not just using the simpler Optimize already defined?
```suggestion
val outer = LocalRelation($"a".int, $"b".int)
val inner = LocalRelation($"x".int, $"y".int)
// NOT EXISTS with FALSE AND correlated_pred - should not throw None.get
val notExistsQuery = outer.where(
Not(Exists(inner.where(Literal.FalseLiteral && $"a" ===
$"x")))).select($"a")
Optimize.execute(notExistsQuery.analyze)
// EXISTS with FALSE AND correlated_pred - should not throw None.get
val existsQuery = outer.where(
Exists(inner.where(Literal.FalseLiteral && $"a" ===
$"x"))).select($"a")
Optimize.execute(existsQuery.analyze)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
