Github user rdblue commented on a diff in the pull request:
https://github.com/apache/spark/pull/21556#discussion_r201763831
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
---
@@ -248,29 +371,29 @@ private[parquet] class ParquetFilters(pushDownDate:
Boolean, pushDownStartWith:
// Probably I missed something and obviously this should be changed.
predicate match {
- case sources.IsNull(name) if canMakeFilterOn(name) =>
+ case sources.IsNull(name) if canMakeFilterOn(name, null) =>
makeEq.lift(nameToType(name)).map(_(name, null))
- case sources.IsNotNull(name) if canMakeFilterOn(name) =>
+ case sources.IsNotNull(name) if canMakeFilterOn(name, null) =>
makeNotEq.lift(nameToType(name)).map(_(name, null))
- case sources.EqualTo(name, value) if canMakeFilterOn(name) =>
+ case sources.EqualTo(name, value) if canMakeFilterOn(name, value) =>
makeEq.lift(nameToType(name)).map(_(name, value))
- case sources.Not(sources.EqualTo(name, value)) if
canMakeFilterOn(name) =>
+ case sources.Not(sources.EqualTo(name, value)) if
canMakeFilterOn(name, value) =>
makeNotEq.lift(nameToType(name)).map(_(name, value))
- case sources.EqualNullSafe(name, value) if canMakeFilterOn(name) =>
+ case sources.EqualNullSafe(name, value) if canMakeFilterOn(name,
value) =>
makeEq.lift(nameToType(name)).map(_(name, value))
- case sources.Not(sources.EqualNullSafe(name, value)) if
canMakeFilterOn(name) =>
+ case sources.Not(sources.EqualNullSafe(name, value)) if
canMakeFilterOn(name, value) =>
makeNotEq.lift(nameToType(name)).map(_(name, value))
--- End diff --
Since makeNotEq is also used for EqualNullSafe, I think it should handle
null values as well.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
