andygrove commented on code in PR #564:
URL: https://github.com/apache/datafusion-comet/pull/564#discussion_r1638975379
##########
spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala:
##########
@@ -1089,36 +1089,148 @@ class CometAggregateSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
- test("covar_pop and covar_samp") {
+ test("covariance") {
withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") {
- Seq("native", "jvm").foreach { cometShuffleMode =>
+ Seq("jvm", "native").foreach { cometShuffleMode =>
withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) {
Seq(true, false).foreach { dictionary =>
withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
- val table = "test"
- withTable(table) {
- sql(
- s"create table $table(col1 int, col2 int, col3 int, col4
float, col5 double," +
- " col6 double, col7 int) using parquet")
- sql(s"insert into $table values(1, 4, null, 1.1, 2.2, null,
1)," +
- " (2, 5, 6, 3.4, 5.6, null, 1), (3, 6, null, 7.9, 2.4, null,
2)")
- val expectedNumOfCometAggregates = 2
- checkSparkAnswerAndNumOfAggregates(
- "SELECT covar_samp(col1, col2), covar_samp(col1, col3),
covar_samp(col4, col5)," +
- " covar_samp(col4, col6) FROM test",
- expectedNumOfCometAggregates)
- checkSparkAnswerAndNumOfAggregates(
- "SELECT covar_pop(col1, col2), covar_pop(col1, col3),
covar_pop(col4, col5)," +
- " covar_pop(col4, col6) FROM test",
- expectedNumOfCometAggregates)
- checkSparkAnswerAndNumOfAggregates(
- "SELECT covar_samp(col1, col2), covar_samp(col1, col3),
covar_samp(col4, col5)," +
- " covar_samp(col4, col6) FROM test GROUP BY col7",
- expectedNumOfCometAggregates)
- checkSparkAnswerAndNumOfAggregates(
- "SELECT covar_pop(col1, col2), covar_pop(col1, col3),
covar_pop(col4, col5)," +
- " covar_pop(col4, col6) FROM test GROUP BY col7",
- expectedNumOfCometAggregates)
+ Seq(true, false).foreach { nullOnDivideByZero =>
+ withSQLConf(
+ "spark.sql.legacy.statisticalAggregate" ->
nullOnDivideByZero.toString) {
+ val table = "test"
+ withTable(table) {
+ sql(
+ s"create table $table(col1 double, col2 double, col3
double) using parquet")
+ sql(s"insert into $table values(1, 4, 1), (2, 5, 1), (3,
6, 2)")
+ val expectedNumOfCometAggregates = 2
+
+ checkSparkAnswerAndNumOfAggregates(
+ s"SELECT covar_samp(col1, col2), covar_pop(col1, col2)
FROM $table",
+ expectedNumOfCometAggregates)
+
+ checkSparkAnswerAndNumOfAggregates(
+ s"SELECT covar_samp(col1, col2), covar_pop(col1, col2)
FROM $table GROUP BY col3",
+ expectedNumOfCometAggregates)
+ }
+
+ withTable(table) {
+ sql(
+ s"create table $table(col1 double, col2 double, col3
double) using parquet")
+ sql(s"insert into $table values(1, 4, 3), (2, -5, 3), (3,
6, 1)")
+ val expectedNumOfCometAggregates = 2
+
+ checkSparkAnswerWithTolAndNumOfAggregates(
+ s"SELECT covar_samp(col1, col2), covar_pop(col1, col2)
FROM $table",
+ expectedNumOfCometAggregates)
+
+ checkSparkAnswerWithTolAndNumOfAggregates(
+ s"SELECT covar_samp(col1, col2), covar_pop(col1, col2)
FROM $table GROUP BY col3",
+ expectedNumOfCometAggregates)
+ }
Review Comment:
It looks like this same block is repeated many times but just the values
change? Perhaps there could be a helper function that can be called with the
different values instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
