Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22857#discussion_r229537395
--- Diff: sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
---
@@ -2585,4 +2585,45 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
checkAnswer(swappedDf.filter($"key"($"map") > "a"), Row(2, Map(2 ->
"b")))
}
+
+ test("SPARK-25860: Replace Literal(null, _) with FalseLiteral whenever
possible") {
+
+ def checkPlanIsEmptyLocalScan(df: DataFrame): Unit =
df.queryExecution.executedPlan match {
+ case s: LocalTableScanExec => assert(s.rows.isEmpty)
+ case p => fail(s"$p is not LocalTableScanExec")
+ }
+
+ val df1 = Seq((1, true), (2, false)).toDF("l", "b")
+ val df2 = Seq(2, 3).toDF("l")
+
+ val q1 = df1.where("IF(l > 10, false, b AND null)")
+ checkAnswer(q1, Seq.empty)
+ checkPlanIsEmptyLocalScan(q1)
+
+ val q2 = df1.where("CASE WHEN l < 10 THEN null WHEN l > 40 THEN false
ELSE null END")
+ checkAnswer(q2, Seq.empty)
+ checkPlanIsEmptyLocalScan(q2)
+
+ val q3 = df1.join(df2, when(df1("l") > df2("l"),
lit(null)).otherwise(df1("b") && lit(null)))
+ checkAnswer(q3, Seq.empty)
+ checkPlanIsEmptyLocalScan(q3)
+
+ val q4 = df1.where("IF(IF(b, null, false), true, null)")
+ checkAnswer(q4, Seq.empty)
+ checkPlanIsEmptyLocalScan(q4)
+
+ val q5 = df1.selectExpr("IF(l > 1 AND null, 5, 1) AS out")
+ checkAnswer(q5, Row(1) :: Row(1) :: Nil)
+ q5.queryExecution.executedPlan.foreach { p =>
+ assert(p.expressions.forall(e => e.find(_.isInstanceOf[If]).isEmpty))
--- End diff --
This test can pass without the optimization. The `ConvertToLocalRelation`
rule will eliminate the `Project`.
Can we use a table as input data? e.g.
```
withTable("t1", "t2") {
Seq((1, true), (2, false)).toDF("l", "b").write.saveAsTable("t1")
Seq(2, 3).toDF("l").write.saveAsTable("t2")
val df1 = spark.table("t1")
val df2 = spark.table("t2")
...
}
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
