urosstan-db commented on code in PR #57714:
URL: https://github.com/apache/spark/pull/57714#discussion_r3711295452
##########
sql/core/src/test/scala/org/apache/spark/sql/jdbc/v2/JDBCV2JoinPushdownIntegrationSuiteBase.scala:
##########
@@ -497,6 +495,55 @@ trait JDBCV2JoinPushdownIntegrationSuiteBase
}
}
+ test("Test aggregate with group by on top of join") {
+ val sqlQuery =
+ s"""
+ |SELECT t1.id, t1.address, min(t2.salary), count(1)
+ |FROM $catalogAndNamespace.$casedJoinTableName1 t1
+ |JOIN $catalogAndNamespace.$casedJoinTableName2 t2 ON t1.id = t2.id
+ |WHERE t1.amount > 1000
+ |GROUP BY t1.id, t1.address
+ |""".stripMargin
+
+ val rowsNoPushdown = withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key
-> "false") {
+ sql(sqlQuery).collect().toSeq
+ }
+
+ assert(rowsNoPushdown.nonEmpty)
+
+ withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key -> "true") {
+ val df = sql(sqlQuery)
+ checkJoinPushed(df)
+ checkAggregateRemoved(df, supportsAggregatePushdown)
+ checkAnswer(df, rowsNoPushdown)
+ }
+ }
+
+ test("Test multi-way join with function in join condition") {
+ val sqlQuery =
+ s"""
+ |SELECT a.id, c.address, d.address, a.amount
+ |FROM $catalogAndNamespace.$casedJoinTableName1 a
+ |JOIN $catalogAndNamespace.$casedJoinTableName1 c
+ | ON a.address = c.address
+ |JOIN $catalogAndNamespace.$casedJoinTableName1 d
+ | ON LOWER(a.address) = LOWER(d.address)
+ |WHERE a.amount >= 1000
+ |""".stripMargin
+
+ val rowsNoPushdown = withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key
-> "false") {
+ sql(sqlQuery).collect().toSeq
+ }
+
+ assert(rowsNoPushdown.nonEmpty)
Review Comment:
I added it as additional guard against modifications on test dataset, so I
would like to preserve some guard against that (to make our test still relevant
in the future), it is not requirement ofc. Do you maybe suggest we make
assertion on entire result then, or asserting on number of rows, etc? Or
totally removing it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
