dbtsai commented on a change in pull request #27728:
[SPARK-25556][SPARK-17636][SPARK-31026][SPARK-31060][SQL][test-hive1.2] Nested
Column Predicate Pushdown for Parquet
URL: https://github.com/apache/spark/pull/27728#discussion_r398738263
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
##########
@@ -187,201 +209,273 @@ abstract class ParquetFilterSuite extends QueryTest
with ParquetTest with Shared
}
test("filter pushdown - boolean") {
- withParquetDataFrame((true :: false :: Nil).map(b =>
Tuple1.apply(Option(b)))) { implicit df =>
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], Seq(Row(true),
Row(false)))
-
- checkFilterPredicate('_1 === true, classOf[Eq[_]], true)
- checkFilterPredicate('_1 <=> true, classOf[Eq[_]], true)
- checkFilterPredicate('_1 =!= true, classOf[NotEq[_]], false)
+ val data = (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val booleanAttr = df(colName).expr
+ assert(df(colName).expr.dataType === BooleanType)
+
+ checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]],
Seq.empty[Row])
+ checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
+ Seq(Row(resultFun(true)), Row(resultFun(false))))
+
+ checkFilterPredicate(booleanAttr === true, classOf[Eq[_]],
resultFun(true))
+ checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]],
resultFun(true))
+ checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]],
resultFun(false))
+ }
}
}
test("filter pushdown - tinyint") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toByte)))) {
implicit df =>
- assert(df.schema.head.dataType === ByteType)
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1.toByte, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1.toByte, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1.toByte, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2.toByte, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3.toByte, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1.toByte, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4.toByte, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1.toByte) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1.toByte) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2.toByte) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3.toByte) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1.toByte) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4.toByte) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4.toByte), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2.toByte || '_1 > 3.toByte,
- classOf[Operators.Or], Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i.toByte)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val tinyIntAttr = df(colName).expr
+ assert(df(colName).expr.dataType === ByteType)
+
+ checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]],
Seq.empty[Row])
+ checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
+ (1 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
+ (2 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]],
resultFun(1))
+ checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]],
resultFun(4))
+
+ checkFilterPredicate(Literal(1.toByte) === tinyIntAttr,
classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr,
classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr,
classOf[LtEq[_]], resultFun(1))
+ checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr,
classOf[GtEq[_]], resultFun(4))
+
+ checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]],
resultFun(4))
+ checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
+ classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
+ }
}
}
test("filter pushdown - smallint") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toShort)))) {
implicit df =>
- assert(df.schema.head.dataType === ShortType)
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1.toShort, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1.toShort, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1.toShort, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2.toShort, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3.toShort, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1.toShort, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4.toShort, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1.toShort) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1.toShort) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2.toShort) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3.toShort) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1.toShort) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4.toShort) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4.toShort), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2.toShort || '_1 > 3.toShort,
- classOf[Operators.Or], Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i.toShort)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val smallIntAttr = df(colName).expr
+ assert(df(colName).expr.dataType === ShortType)
+
+ checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]],
Seq.empty[Row])
+ checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
+ (1 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
+ (2 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]],
resultFun(1))
+ checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]],
resultFun(4))
+
+ checkFilterPredicate(Literal(1.toShort) === smallIntAttr,
classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr,
classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(Literal(2.toShort) > smallIntAttr,
classOf[Lt[_]], resultFun(1))
+ checkFilterPredicate(Literal(3.toShort) < smallIntAttr,
classOf[Gt[_]], resultFun(4))
+ checkFilterPredicate(Literal(1.toShort) >= smallIntAttr,
classOf[LtEq[_]], resultFun(1))
+ checkFilterPredicate(Literal(4.toShort) <= smallIntAttr,
classOf[GtEq[_]], resultFun(4))
+
+ checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]],
resultFun(4))
+ checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr >
3.toShort,
+ classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
+ }
}
}
test("filter pushdown - integer") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or],
Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val intAttr = df(colName).expr
+ assert(df(colName).expr.dataType === IntegerType)
+
+ checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+ checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
+ (1 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
+ (2 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
+ checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
+ checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
+ checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+ checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]],
resultFun(4))
+
+ checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
+ checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
+ Seq(Row(resultFun(1)), Row(resultFun(4))))
+ }
}
}
test("filter pushdown - long") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) {
implicit df =>
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or],
Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i.toLong)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val longAttr = df(colName).expr
+ assert(df(colName).expr.dataType === LongType)
+
+ checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+ checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
+ (1 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
+ (2 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
+ checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
+ checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
+ checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+ checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]],
resultFun(4))
+
+ checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
+ checkFilterPredicate(longAttr < 2 || longAttr > 3,
classOf[Operators.Or],
+ Seq(Row(resultFun(1)), Row(resultFun(4))))
+ }
}
}
test("filter pushdown - float") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) {
implicit df =>
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or],
Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i.toFloat)))
+ import testImplicits._
+ withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+ withParquetDataFrame(inputDF) { implicit df =>
+ val floatAttr = df(colName).expr
+ assert(df(colName).expr.dataType === FloatType)
+
+ checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+ checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
+ (1 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
+ checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
+ (2 to 4).map(i => Row.apply(resultFun(i))))
+
+ checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
+ checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
+ checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
+ checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+ checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]],
resultFun(1))
+ checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]],
resultFun(4))
+ checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]],
resultFun(1))
+ checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]],
resultFun(4))
+
+ checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
+ checkFilterPredicate(floatAttr < 2 || floatAttr > 3,
classOf[Operators.Or],
+ Seq(Row(resultFun(1)), Row(resultFun(4))))
+ }
}
}
test("filter pushdown - double") {
- withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) {
implicit df =>
- checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
- checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
- checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to
4).map(Row.apply(_)))
-
- checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
- checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
- checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
- checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
- checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
- checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
- checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
- checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
- checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
- checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or],
Seq(Row(1), Row(4)))
+ val data = (1 to 4).map(i => Tuple1(Option(i.toDouble)))
+ import testImplicits._
Review comment:
Unfortunately, this implicit import will fail
```scala
} { implicit df =>
checkFilterPredicate(
'_1.startsWith("blah").asInstanceOf[Predicate],
classOf[UserDefinedByInstance[_, _]],
```
More specifically, the implicit conversion from `Symbols` to `Expression`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
