This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new e67a6fc81c07 [SPARK-49488][SQL][FOLLOWUP] Do not push down extract
expression if extracted field is second
e67a6fc81c07 is described below
commit e67a6fc81c07e941fdc0e46fe78eb485f2b303fb
Author: beliefer <belie...@163.com>
AuthorDate: Tue Apr 22 08:09:42 2025 +0800
[SPARK-49488][SQL][FOLLOWUP] Do not push down extract expression if
extracted field is second
### What changes were proposed in this pull request?
This PR aims to forbid the pushdown while extract `second` from date or
interval.
### Why are the changes needed?
Currently, Spark supports extract second from date or interval and it
returns result with `decimal(8, 6)`.
But extract second from date or interval returns integer in MySQL.
So we should forbid the pushdown.
### Does this PR introduce _any_ user-facing change?
'No'.
### How was this patch tested?
GA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50637 from beliefer/SPARK-49488_f2.
Authored-by: beliefer <belie...@163.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
(cherry picked from commit c77b163e4cb34ec89db975d78c03361b3bab57a8)
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala | 11 ++++++++++-
.../main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala | 8 ++++++--
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
index 3c63e17a953c..dab74a01e3f9 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
@@ -188,7 +188,7 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationV2Suite with V2JDBCTest
assert(rows2(0).getString(0) === "amy")
assert(rows2(1).getString(0) === "alex")
- val df3 = sql(s"SELECT name FROM $tbl WHERE second(time1) = 0 AND
month(date1) = 5")
+ val df3 = sql(s"SELECT name FROM $tbl WHERE month(date1) = 5")
checkFilterPushed(df3)
val rows3 = df3.collect()
assert(rows3.length === 2)
@@ -255,6 +255,15 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationV2Suite with V2JDBCTest
assert(rows(0).getString(0) === "tom")
}
+ withClue("second") {
+ val df = sql(s"SELECT name FROM $tbl WHERE second(time1) = 0 AND
month(date1) = 5")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 2)
+ assert(rows(0).getString(0) === "amy")
+ assert(rows(1).getString(0) === "alex")
+ }
+
val df9 = sql(s"SELECT name FROM $tbl WHERE " +
"dayofyear(date1) > 100 order by dayofyear(date1) limit 1")
checkFilterPushed(df9)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
index 5dec15b0fbcd..4323fa4ed99b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
@@ -57,11 +57,15 @@ private case class MySQLDialect() extends JdbcDialect with
SQLConfHelper with No
field match {
case "DAY_OF_YEAR" => s"DAYOFYEAR(${build(extract.source())})"
case "WEEK" => s"WEEKOFYEAR(${build(extract.source())})"
- case "YEAR_OF_WEEK" => visitUnexpectedExpr(extract)
+ // MySQL does not support the date field YEAR_OF_WEEK.
+ // We can't push down SECOND due to the difference in result types
between Spark and
+ // MySQL. Spark returns decimal(8, 6), but MySQL returns integer.
+ case "YEAR_OF_WEEK" | "SECOND" =>
+ visitUnexpectedExpr(extract)
// WEEKDAY uses Monday = 0, Tuesday = 1, ... and ISO standard is
Monday = 1, ...,
// so we use the formula (WEEKDAY + 1) to follow the ISO standard.
case "DAY_OF_WEEK" => s"(WEEKDAY(${build(extract.source())}) + 1)"
- // SECOND, MINUTE, HOUR, DAY, MONTH, QUARTER, YEAR are identical on
MySQL and Spark for
+ // MINUTE, HOUR, DAY, MONTH, QUARTER, YEAR are identical on MySQL and
Spark for
// both datetime and interval types.
case _ => super.visitExtract(field, build(extract.source()))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
