kazuyukitanimura commented on code in PR #1343:
URL: https://github.com/apache/datafusion-comet/pull/1343#discussion_r1943711444
##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -2387,6 +2387,8 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde with CometExprShim
case _: ArrayIntersect => convert(CometArrayIntersect)
case _: ArrayJoin => convert(CometArrayJoin)
case _: ArraysOverlap => convert(CometArraysOverlap)
+ case _ if expr.prettyName == "array_except" =>
Review Comment:
Is `ArrayExcept` available only for newer version?
##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -126,6 +126,21 @@ object CometArraysOverlap extends CometExpressionSerde
with IncompatExpr {
}
}
+object CometArrayExcept extends CometExpressionSerde with CometExprShim {
Review Comment:
Any reason for using `CometExprShim`?
Also not sure how ready this is regarding Spark compatibility although it
seems the test coverage is good. Adding `IncompatExpr` might be an option.
##########
spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala:
##########
@@ -292,4 +292,89 @@ class CometArrayExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelp
}
}
+ test("array_except - basic test (only integer values)") {
+ withSQLConf(CometConf.COMET_EXPR_ALLOW_INCOMPATIBLE.key -> "true") {
+ Seq(true, false).foreach { dictionaryEnabled =>
+ withTempDir { dir =>
+ val path = new Path(dir.toURI.toString, "test.parquet")
+ makeParquetFileAllTypes(path, dictionaryEnabled, 10000)
+ spark.read.parquet(path.toString).createOrReplaceTempView("t1")
+
+ checkSparkAnswerAndOperator(
+ sql("SELECT array_except(array(_2, _3, _4), array(_3, _4)) from
t1"))
+ checkSparkAnswerAndOperator(sql("SELECT array_except(array(_18),
array(_19)) from t1"))
+ checkSparkAnswerAndOperator(spark.sql(
+ "SELECT array_except((CASE WHEN _2 = _3 THEN array(_2, _3, _4)
END), array(_4)) FROM t1"))
Review Comment:
`_2 = _3` would be always true?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
