uros-b commented on code in PR #57714:
URL: https://github.com/apache/spark/pull/57714#discussion_r3703572467
##########
sql/core/src/test/scala/org/apache/spark/sql/jdbc/v2/JDBCV2JoinPushdownIntegrationSuiteBase.scala:
##########
@@ -497,6 +495,55 @@ trait JDBCV2JoinPushdownIntegrationSuiteBase
}
}
+ test("Test aggregate with group by on top of join") {
+ val sqlQuery =
+ s"""
+ |SELECT t1.id, t1.address, min(t2.salary), count(1)
+ |FROM $catalogAndNamespace.$casedJoinTableName1 t1
+ |JOIN $catalogAndNamespace.$casedJoinTableName2 t2 ON t1.id = t2.id
+ |WHERE t1.amount > 1000
+ |GROUP BY t1.id, t1.address
+ |""".stripMargin
+
+ val rowsNoPushdown = withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key
-> "false") {
+ sql(sqlQuery).collect().toSeq
+ }
+
+ assert(rowsNoPushdown.nonEmpty)
+
+ withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key -> "true") {
+ val df = sql(sqlQuery)
+ checkJoinPushed(df)
+ checkAggregateRemoved(df, supportsAggregatePushdown)
+ checkAnswer(df, rowsNoPushdown)
+ }
+ }
+
+ test("Test multi-way join with function in join condition") {
+ val sqlQuery =
+ s"""
+ |SELECT a.id, c.address, d.address, a.amount
+ |FROM $catalogAndNamespace.$casedJoinTableName1 a
+ |JOIN $catalogAndNamespace.$casedJoinTableName1 c
+ | ON a.address = c.address
+ |JOIN $catalogAndNamespace.$casedJoinTableName1 d
+ | ON LOWER(a.address) = LOWER(d.address)
+ |WHERE a.amount >= 1000
+ |""".stripMargin
+
+ val rowsNoPushdown = withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key
-> "false") {
+ sql(sqlQuery).collect().toSeq
+ }
+
+ assert(rowsNoPushdown.nonEmpty)
Review Comment:
These asserts don't seem to be doing much at the moment, please consider
removing them.
##########
sql/core/src/test/scala/org/apache/spark/sql/jdbc/v2/JDBCV2JoinPushdownIntegrationSuiteBase.scala:
##########
@@ -497,6 +495,55 @@ trait JDBCV2JoinPushdownIntegrationSuiteBase
}
}
+ test("Test aggregate with group by on top of join") {
+ val sqlQuery =
+ s"""
+ |SELECT t1.id, t1.address, min(t2.salary), count(1)
+ |FROM $catalogAndNamespace.$casedJoinTableName1 t1
+ |JOIN $catalogAndNamespace.$casedJoinTableName2 t2 ON t1.id = t2.id
+ |WHERE t1.amount > 1000
+ |GROUP BY t1.id, t1.address
+ |""".stripMargin
+
+ val rowsNoPushdown = withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key
-> "false") {
+ sql(sqlQuery).collect().toSeq
+ }
+
+ assert(rowsNoPushdown.nonEmpty)
Review Comment:
Same as https://github.com/apache/spark/pull/57714/changes#r3703572467.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
