This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5d52bb36d3b [SPARK-41486][SQL][TESTS] Upgrade `MySQL` docker image to
8.0.31 to support `ARM64` test
5d52bb36d3b is described below
commit 5d52bb36d3ba7e619169289f8a576db5ea10990f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Dec 11 22:23:17 2022 -0800
[SPARK-41486][SQL][TESTS] Upgrade `MySQL` docker image to 8.0.31 to support
`ARM64` test
### What changes were proposed in this pull request?
This PR aims to upgrade MySQL JDBC Integration Test docker image from
5.7.36 to 8.0.31 to support ARM64 testing.
### Why are the changes needed?
Since MySQL 8.0, MySQL provides multi-arch images. After this upgrade, we
can run MySQL JDBC Integration Test on ARM64 environment.
### Does this PR introduce _any_ user-facing change?
No, this is a test code PR.
### How was this patch tested?
- Pass GitHub Action Ci for AMD64 architecture
- Manually run MySQLIntegrationSuites on ARM64 architecture.
I manually run this PR on Apple Silicon.
```
$ ./build/sbt -Pdocker-integration-tests "testOnly *MySQLIntegrationSuite"
[info] MySQLIntegrationSuite:
[info] - SPARK-33034: ALTER TABLE ... add new columns (4 seconds, 766
milliseconds)
[info] - SPARK-33034: ALTER TABLE ... drop column (389 milliseconds)
[info] - SPARK-33034: ALTER TABLE ... update column type (285 milliseconds)
[info] - SPARK-33034: ALTER TABLE ... rename column (206 milliseconds)
[info] - SPARK-33034: ALTER TABLE ... update column nullability (167
milliseconds)
[info] - CREATE TABLE with table comment (98 milliseconds)
[info] - CREATE TABLE with table property (113 milliseconds)
[info] - SPARK-36895: Test INDEX Using SQL (559 milliseconds)
[info] - SPARK-37038: Test TABLESAMPLE (0 milliseconds)
[info] - scan with aggregate push-down: VAR_POP without DISTINCT (3
seconds, 718 milliseconds)
[info] - scan with aggregate push-down: VAR_SAMP without DISTINCT (602
milliseconds)
[info] - scan with aggregate push-down: STDDEV_POP without DISTINCT (624
milliseconds)
[info] - scan with aggregate push-down: STDDEV_SAMP without DISTINCT (390
milliseconds)
[info] MySQLIntegrationSuite:
[info] Passed: Total 0, Failed 0, Errors 0, Passed 0
[info] No tests to run for mllib / Test / testOnly
[info] - Basic test (432 milliseconds)
[info] - Numeric types (324 milliseconds)
[info] - Date types (871 milliseconds)
[info] - String types (302 milliseconds)
[info] - Basic write test (1 second, 341 milliseconds)
[info] - query JDBC option (334 milliseconds)
[info] Run completed in 1 minute, 15 seconds.
[info] Total number of tests run: 19
[info] Suites: completed 2, aborted 0
[info] Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
$ ./build/sbt -Pdocker-integration-tests "testOnly *MySQLNamespaceSuite"
[info] MySQLNamespaceSuite:
[info] - listNamespaces: basic behavior (1 second, 367 milliseconds)
[info] - Drop namespace (231 milliseconds)
[info] - Create or remove comment of namespace unsupported (186
milliseconds)
[info] Run completed in 33 seconds, 456 milliseconds.
[info] Total number of tests run: 3
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 3, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```
Closes #39029 from dongjoon-hyun/SPARK-41486.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala | 6 +++---
.../scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala | 6 +++---
.../scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala | 6 +++---
.../scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala | 3 ++-
4 files changed, 11 insertions(+), 10 deletions(-)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index 2bde50d01eb..bc202b1b832 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -26,9 +26,9 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
import org.apache.spark.tags.DockerTest
/**
- * To run this test suite for a specific version (e.g., mysql:5.7.36):
+ * To run this test suite for a specific version (e.g., mysql:8.0.31):
* {{{
- * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31
* ./build/sbt -Pdocker-integration-tests
* "testOnly org.apache.spark.sql.jdbc.MySQLIntegrationSuite"
* }}}
@@ -36,7 +36,7 @@ import org.apache.spark.tags.DockerTest
@DockerTest
class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
override val db = new DatabaseOnDocker {
- override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:5.7.36")
+ override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:8.0.31")
override val env = Map(
"MYSQL_ROOT_PASSWORD" -> "rootpass"
)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
index 6e76b74c7d8..072fdbb3f34 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala
@@ -29,9 +29,9 @@ import org.apache.spark.sql.types._
import org.apache.spark.tags.DockerTest
/**
- * To run this test suite for a specific version (e.g., mysql:5.7.36):
+ * To run this test suite for a specific version (e.g., mysql:8.0.31):
* {{{
- * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31
* ./build/sbt -Pdocker-integration-tests "testOnly
*v2*MySQLIntegrationSuite"
* }}}
*/
@@ -39,7 +39,7 @@ import org.apache.spark.tags.DockerTest
class MySQLIntegrationSuite extends DockerJDBCIntegrationV2Suite with
V2JDBCTest {
override val catalogName: String = "mysql"
override val db = new DatabaseOnDocker {
- override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:5.7.36")
+ override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:8.0.31")
override val env = Map(
"MYSQL_ROOT_PASSWORD" -> "rootpass"
)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala
index d8dee61d70e..b73e2b8fd23 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala
@@ -28,16 +28,16 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.tags.DockerTest
/**
- * To run this test suite for a specific version (e.g., mysql:5.7.36):
+ * To run this test suite for a specific version (e.g., mysql:8.0.31):
* {{{
- * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31
* ./build/sbt -Pdocker-integration-tests "testOnly
*v2*MySQLNamespaceSuite"
* }}}
*/
@DockerTest
class MySQLNamespaceSuite extends DockerJDBCIntegrationSuite with
V2JDBCNamespaceTest {
override val db = new DatabaseOnDocker {
- override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:5.7.36")
+ override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME",
"mysql:8.0.31")
override val env = Map(
"MYSQL_ROOT_PASSWORD" -> "rootpass"
)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
index 50e79e03a7a..b2500a2dbf2 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala
@@ -62,7 +62,8 @@ private[v2] trait V2JDBCNamespaceTest extends
SharedSparkSession with DockerInte
Map.empty[String, String]
}
catalog.createNamespace(Array("foo"), commentMap.asJava)
- assert(catalog.listNamespaces() === listNamespaces(Array("foo")))
+ assert(catalog.listNamespaces().map(_.toSet).toSet ===
+ listNamespaces(Array("foo")).map(_.toSet).toSet)
assert(catalog.listNamespaces(Array("foo")) === Array())
assert(catalog.namespaceExists(Array("foo")) === true)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
