This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 999abb5671ce [SPARK-47186][DOCKER][TESTS] Add some timeouts options and logs to improve the debuggability for docker integration test 999abb5671ce is described below commit 999abb5671ce54fcac6cfba365d61e9f47ba7733 Author: Kent Yao <y...@apache.org> AuthorDate: Wed Feb 28 10:25:05 2024 +0800 [SPARK-47186][DOCKER][TESTS] Add some timeouts options and logs to improve the debuggability for docker integration test ### What changes were proposed in this pull request? This PR adds test-scoped options: - Timeout for pulling the Docker image before the tests start. - `spark.test.docker.imagePullTimeout` - Timeout for container to spin up. - `spark.test.docker.startContainerTimeout` - Timeout for connecting the inner service in the container - `spark.test.docker.connectionTimeout` This PR also adds loggings(excluding the downloading/extracting details) for the imaging pulling step which is time-consuming: ``` 24/02/27 19:03:17.112 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling from gvenzl/oracle-free 23.3-slim 24/02/27 19:03:17.112 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer 5cbb6d705282 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer f1544b3116d0 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer 1dff807126c4 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer 603266ad0104 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer 10f286d1795c 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pulling fs layer 7c4de5471fcf 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Waiting 603266ad0104 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Waiting 10f286d1795c 24/02/27 19:03:17.113 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Waiting 7c4de5471fcf 24/02/27 19:03:59.725 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum 5cbb6d705282 24/02/27 19:03:59.725 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete 5cbb6d705282 24/02/27 19:04:12.512 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Extracting 5cbb6d705282 62.3 MiB 24/02/27 19:04:12.801 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pull complete 5cbb6d705282 24/02/27 19:04:25.905 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum f1544b3116d0 24/02/27 19:04:25.906 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete f1544b3116d0 24/02/27 19:04:39.533 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Extracting f1544b3116d0 103.5 MiB 24/02/27 19:04:39.647 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pull complete f1544b3116d0 24/02/27 19:04:46.451 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum 10f286d1795c 24/02/27 19:04:46.452 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete 10f286d1795c 24/02/27 19:05:39.623 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum 7c4de5471fcf 24/02/27 19:05:39.623 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete 7c4de5471fcf 24/02/27 19:05:40.889 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum 1dff807126c4 24/02/27 19:05:40.890 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete 1dff807126c4 24/02/27 19:05:51.976 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Verifying Checksum 603266ad0104 24/02/27 19:05:51.976 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Download complete 603266ad0104 24/02/27 19:05:59.357 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Extracting 1dff807126c4 178.3 MiB 24/02/27 19:05:59.429 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pull complete 1dff807126c4 24/02/27 19:06:10.751 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Extracting 603266ad0104 110.2 MiB 24/02/27 19:06:11.117 docker-java-stream--1665796424 INFO OracleIntegrationSuite: Pull complete 603266ad0104 ``` ### Why are the changes needed? Some districts might suffer from network issues with the official docker registry ### Does this PR introduce _any_ user-facing change? no, dev-only ### How was this patch tested? docker it ### Was this patch authored or co-authored using generative AI tooling? no Closes #45284 from yaooqinn/SPARK-47186. Authored-by: Kent Yao <y...@apache.org> Signed-off-by: Kent Yao <y...@apache.org> --- connector/docker-integration-tests/README.md | 21 +++++++++ .../sql/jdbc/DockerJDBCIntegrationSuite.scala | 52 +++++++++++++++++++--- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/connector/docker-integration-tests/README.md b/connector/docker-integration-tests/README.md index 61501be91ca2..0192947bdbf9 100644 --- a/connector/docker-integration-tests/README.md +++ b/connector/docker-integration-tests/README.md @@ -108,4 +108,25 @@ The following are the available properties that can be passed to optimize testin </td> <td>true</td> </tr> + <tr> + <td><code>spark.test.docker.imagePullTimeout</code></td> + <td> + Timeout for pulling the Docker image before the tests start. + </td> + <td>5min</td> + </tr> + <tr> + <td><code>spark.test.docker.startContainerTimeout</code></td> + <td> + Timeout for container to spin up. + </td> + <td>5min</td> + </tr> + <tr> + <td><code>spark.test.docker.connectionTimeout</code></td> + <td> + Timeout for connecting the inner service in the container, such as JDBC services. + </td> + <td>5min(might get overridden by some inherits)</td> + </tr> </table> diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala index a427648b2aa4..806db5b6c757 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala @@ -22,21 +22,23 @@ import java.sql.{Connection, DriverManager} import java.util.Properties import java.util.concurrent.TimeUnit +import scala.concurrent.TimeoutException import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.async.{ResultCallback, ResultCallbackTemplate} -import com.github.dockerjava.api.command.CreateContainerResponse +import com.github.dockerjava.api.command.{CreateContainerResponse, PullImageResultCallback} import com.github.dockerjava.api.exception.NotFoundException import com.github.dockerjava.api.model._ import com.github.dockerjava.core.{DefaultDockerClientConfig, DockerClientImpl} import com.github.dockerjava.zerodep.ZerodepDockerHttpClient -import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.SpanSugar._ import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.DockerUtils +import org.apache.spark.util.Utils.{bytesToString, timeStringAsSeconds} abstract class DatabaseOnDocker { /** @@ -101,11 +103,18 @@ abstract class DockerJDBCIntegrationSuite protected val dockerIp = DockerUtils.getDockerIp() val db: DatabaseOnDocker - val connectionTimeout = timeout(5.minutes) val keepContainer = sys.props.getOrElse("spark.test.docker.keepContainer", "false").toBoolean val removePulledImage = sys.props.getOrElse("spark.test.docker.removePulledImage", "true").toBoolean + protected val imagePullTimeout: Long = + timeStringAsSeconds(sys.props.getOrElse("spark.test.docker.imagePullTimeout", "5min")) + protected val startContainerTimeout: Long = + timeStringAsSeconds(sys.props.getOrElse("spark.test.docker.startContainerTimeout", "5min")) + protected val connectionTimeout: PatienceConfiguration.Timeout = { + val timeoutStr = sys.props.getOrElse("spark.test.docker.conn", "5min") + timeout(timeStringAsSeconds(timeoutStr).seconds) + } private var docker: DockerClient = _ // Configure networking (necessary for boot2docker / Docker Machine) @@ -142,10 +151,39 @@ abstract class DockerJDBCIntegrationSuite } catch { case e: NotFoundException => log.warn(s"Docker image ${db.imageName} not found; pulling image from registry") + val callback = new PullImageResultCallback { + override def onNext(item: PullResponseItem): Unit = { + super.onNext(item) + if (item.getStatus != null) { + item.getStatus match { + case s if item.getProgressDetail != null && + item.getProgressDetail.getCurrent != null && + item.getProgressDetail.getCurrent == item.getProgressDetail.getTotal => + // logging for final progress procedural status + logInfo(s"$s ${item.getId} ${bytesToString(item.getProgressDetail.getTotal)}") + case s if s != "Downloading" && s != "Extracting" => + logInfo(s"${item.getStatus} ${item.getId}") + case _ => + } + } + } + + override def onComplete(): Unit = { + pulled = true + } + + override def onError(throwable: Throwable): Unit = { + logError(s"Failed to pull Docker image ${db.imageName}", throwable) + } + } + docker.pullImageCmd(db.imageName) - .start() - .awaitCompletion(connectionTimeout.value.toSeconds, TimeUnit.SECONDS) - pulled = true + .exec(callback) + .awaitCompletion(imagePullTimeout, TimeUnit.SECONDS) + if (!pulled) { + throw new TimeoutException( + s"Timeout('$imagePullTimeout secs') waiting for image ${db.imageName} to be pulled") + } } val hostConfig = HostConfig @@ -176,7 +214,7 @@ abstract class DockerJDBCIntegrationSuite container = createContainerCmd.exec() // Start the container and wait until the database can accept JDBC connections: docker.startContainerCmd(container.getId).exec() - eventually(connectionTimeout, interval(1.second)) { + eventually(timeout(startContainerTimeout.seconds), interval(1.second)) { val response = docker.inspectContainerCmd(container.getId).exec() assert(response.getState.getRunning) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org