This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 999abb5671ce [SPARK-47186][DOCKER][TESTS] Add some timeouts options
and logs to improve the debuggability for docker integration test
999abb5671ce is described below
commit 999abb5671ce54fcac6cfba365d61e9f47ba7733
Author: Kent Yao <y...@apache.org>
AuthorDate: Wed Feb 28 10:25:05 2024 +0800
[SPARK-47186][DOCKER][TESTS] Add some timeouts options and logs to improve
the debuggability for docker integration test
### What changes were proposed in this pull request?
This PR adds test-scoped options:
- Timeout for pulling the Docker image before the tests start. -
`spark.test.docker.imagePullTimeout`
- Timeout for container to spin up. -
`spark.test.docker.startContainerTimeout`
- Timeout for connecting the inner service in the container -
`spark.test.docker.connectionTimeout`
This PR also adds loggings(excluding the downloading/extracting details)
for the imaging pulling step which is time-consuming:
```
24/02/27 19:03:17.112 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling from gvenzl/oracle-free 23.3-slim
24/02/27 19:03:17.112 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer 5cbb6d705282
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer f1544b3116d0
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer 1dff807126c4
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer 603266ad0104
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer 10f286d1795c
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pulling fs layer 7c4de5471fcf
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Waiting 603266ad0104
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Waiting 10f286d1795c
24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Waiting 7c4de5471fcf
24/02/27 19:03:59.725 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum 5cbb6d705282
24/02/27 19:03:59.725 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete 5cbb6d705282
24/02/27 19:04:12.512 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Extracting 5cbb6d705282 62.3 MiB
24/02/27 19:04:12.801 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pull complete 5cbb6d705282
24/02/27 19:04:25.905 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum f1544b3116d0
24/02/27 19:04:25.906 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete f1544b3116d0
24/02/27 19:04:39.533 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Extracting f1544b3116d0 103.5 MiB
24/02/27 19:04:39.647 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pull complete f1544b3116d0
24/02/27 19:04:46.451 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum 10f286d1795c
24/02/27 19:04:46.452 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete 10f286d1795c
24/02/27 19:05:39.623 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum 7c4de5471fcf
24/02/27 19:05:39.623 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete 7c4de5471fcf
24/02/27 19:05:40.889 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum 1dff807126c4
24/02/27 19:05:40.890 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete 1dff807126c4
24/02/27 19:05:51.976 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Verifying Checksum 603266ad0104
24/02/27 19:05:51.976 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Download complete 603266ad0104
24/02/27 19:05:59.357 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Extracting 1dff807126c4 178.3 MiB
24/02/27 19:05:59.429 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pull complete 1dff807126c4
24/02/27 19:06:10.751 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Extracting 603266ad0104 110.2 MiB
24/02/27 19:06:11.117 docker-java-stream--1665796424 INFO
OracleIntegrationSuite: Pull complete 603266ad0104
```
### Why are the changes needed?
Some districts might suffer from network issues with the official docker
registry
### Does this PR introduce _any_ user-facing change?
no, dev-only
### How was this patch tested?
docker it
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #45284 from yaooqinn/SPARK-47186.
Authored-by: Kent Yao <y...@apache.org>
Signed-off-by: Kent Yao <y...@apache.org>
---
connector/docker-integration-tests/README.md | 21 +++++++++
.../sql/jdbc/DockerJDBCIntegrationSuite.scala | 52 +++++++++++++++++++---
2 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/connector/docker-integration-tests/README.md
b/connector/docker-integration-tests/README.md
index 61501be91ca2..0192947bdbf9 100644
--- a/connector/docker-integration-tests/README.md
+++ b/connector/docker-integration-tests/README.md
@@ -108,4 +108,25 @@ The following are the available properties that can be
passed to optimize testin
</td>
<td>true</td>
</tr>
+ <tr>
+ <td><code>spark.test.docker.imagePullTimeout</code></td>
+ <td>
+ Timeout for pulling the Docker image before the tests start.
+ </td>
+ <td>5min</td>
+ </tr>
+ <tr>
+ <td><code>spark.test.docker.startContainerTimeout</code></td>
+ <td>
+ Timeout for container to spin up.
+ </td>
+ <td>5min</td>
+ </tr>
+ <tr>
+ <td><code>spark.test.docker.connectionTimeout</code></td>
+ <td>
+ Timeout for connecting the inner service in the container, such as JDBC
services.
+ </td>
+ <td>5min(might get overridden by some inherits)</td>
+ </tr>
</table>
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index a427648b2aa4..806db5b6c757 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -22,21 +22,23 @@ import java.sql.{Connection, DriverManager}
import java.util.Properties
import java.util.concurrent.TimeUnit
+import scala.concurrent.TimeoutException
import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal
import com.github.dockerjava.api.DockerClient
import com.github.dockerjava.api.async.{ResultCallback, ResultCallbackTemplate}
-import com.github.dockerjava.api.command.CreateContainerResponse
+import com.github.dockerjava.api.command.{CreateContainerResponse,
PullImageResultCallback}
import com.github.dockerjava.api.exception.NotFoundException
import com.github.dockerjava.api.model._
import com.github.dockerjava.core.{DefaultDockerClientConfig, DockerClientImpl}
import com.github.dockerjava.zerodep.ZerodepDockerHttpClient
-import org.scalatest.concurrent.Eventually
+import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.SpanSugar._
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.util.DockerUtils
+import org.apache.spark.util.Utils.{bytesToString, timeStringAsSeconds}
abstract class DatabaseOnDocker {
/**
@@ -101,11 +103,18 @@ abstract class DockerJDBCIntegrationSuite
protected val dockerIp = DockerUtils.getDockerIp()
val db: DatabaseOnDocker
- val connectionTimeout = timeout(5.minutes)
val keepContainer =
sys.props.getOrElse("spark.test.docker.keepContainer", "false").toBoolean
val removePulledImage =
sys.props.getOrElse("spark.test.docker.removePulledImage",
"true").toBoolean
+ protected val imagePullTimeout: Long =
+
timeStringAsSeconds(sys.props.getOrElse("spark.test.docker.imagePullTimeout",
"5min"))
+ protected val startContainerTimeout: Long =
+
timeStringAsSeconds(sys.props.getOrElse("spark.test.docker.startContainerTimeout",
"5min"))
+ protected val connectionTimeout: PatienceConfiguration.Timeout = {
+ val timeoutStr = sys.props.getOrElse("spark.test.docker.conn", "5min")
+ timeout(timeStringAsSeconds(timeoutStr).seconds)
+ }
private var docker: DockerClient = _
// Configure networking (necessary for boot2docker / Docker Machine)
@@ -142,10 +151,39 @@ abstract class DockerJDBCIntegrationSuite
} catch {
case e: NotFoundException =>
log.warn(s"Docker image ${db.imageName} not found; pulling image
from registry")
+ val callback = new PullImageResultCallback {
+ override def onNext(item: PullResponseItem): Unit = {
+ super.onNext(item)
+ if (item.getStatus != null) {
+ item.getStatus match {
+ case s if item.getProgressDetail != null &&
+ item.getProgressDetail.getCurrent != null &&
+ item.getProgressDetail.getCurrent ==
item.getProgressDetail.getTotal =>
+ // logging for final progress procedural status
+ logInfo(s"$s ${item.getId}
${bytesToString(item.getProgressDetail.getTotal)}")
+ case s if s != "Downloading" && s != "Extracting" =>
+ logInfo(s"${item.getStatus} ${item.getId}")
+ case _ =>
+ }
+ }
+ }
+
+ override def onComplete(): Unit = {
+ pulled = true
+ }
+
+ override def onError(throwable: Throwable): Unit = {
+ logError(s"Failed to pull Docker image ${db.imageName}",
throwable)
+ }
+ }
+
docker.pullImageCmd(db.imageName)
- .start()
- .awaitCompletion(connectionTimeout.value.toSeconds,
TimeUnit.SECONDS)
- pulled = true
+ .exec(callback)
+ .awaitCompletion(imagePullTimeout, TimeUnit.SECONDS)
+ if (!pulled) {
+ throw new TimeoutException(
+ s"Timeout('$imagePullTimeout secs') waiting for image
${db.imageName} to be pulled")
+ }
}
val hostConfig = HostConfig
@@ -176,7 +214,7 @@ abstract class DockerJDBCIntegrationSuite
container = createContainerCmd.exec()
// Start the container and wait until the database can accept JDBC
connections:
docker.startContainerCmd(container.getId).exec()
- eventually(connectionTimeout, interval(1.second)) {
+ eventually(timeout(startContainerTimeout.seconds), interval(1.second)) {
val response = docker.inspectContainerCmd(container.getId).exec()
assert(response.getState.getRunning)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
