This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 166cd5a YETUS-1139. Option to limit docker build output (#237)
166cd5a is described below
commit 166cd5af61e5ea77f51fa34e3b4a0b7c94172cd3
Author: Allen Wittenauer <[email protected]>
AuthorDate: Tue Nov 16 12:50:58 2021 -0800
YETUS-1139. Option to limit docker build output (#237)
---
.../in-progress/precommit/docker.html.md | 3 ++
.../in-progress/precommit/robots/travisci.html.md | 1 +
precommit/src/main/shell/core.d/docker.sh | 43 +++++++++++++++++-----
precommit/src/main/shell/robots.d/travisci.sh | 5 +++
4 files changed, 42 insertions(+), 10 deletions(-)
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
b/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
index 24ca7f8..9beb22f 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
@@ -59,6 +59,9 @@ The `--dockerfile` parameter allows one to provide a custom
Dockerfile instead.
Dockerfile images will be named with a test-patch prefix and suffix with
either a date or a git commit hash. By using this information, test-patch will
automatically manage broken/stale container images that are hanging around if
it is run in `--robot` mode. In this way, if Docker fails to build the image,
the disk space should eventually be cleaned and returned back to the system.
The docker mode can also be run in a "safe" mode that prevents deletions via
the `--dockerdelrep` option. [...]
+If you are using a system such as [Travis CI](../robots/travisci) that has
strict limits on logging, the `--docker-build-output`
+option can control whether the `docker build` process is sent to the screen.
+
### COPY and ADD in Dockerfiles
In order to use both 'YETUS CUT HERE' and a Dockerfile that uses COPY and ADD
directives, the Docker API must be version 18 or higher. If the API version is
17 or lower, the Dockerfile will be copied to a temporary directory to be
processed, thus removing the Docker build context in the process.
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
index 23fdead..7630788 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
@@ -32,6 +32,7 @@ images without using `docker run` in the before_install
phase. Therefore, using
in the [Apache Yetus Docker Hub Images](../../../../../yetus-docker-image)
page.
NOTE: As of this writing (2020-10-31), Travis CI does not support Docker
BuildKit. It is disabled by default.
+ Also, docker build logs are automatically sent to a file due to the Travis
CI logging limits.
See also:
diff --git a/precommit/src/main/shell/core.d/docker.sh
b/precommit/src/main/shell/core.d/docker.sh
index b5b0387..cec18b9 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -23,6 +23,7 @@ DOCKER_DESTRUCTIVE=true
DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
DOCKERSUPPORT=false
DOCKER_ENABLE_PRIVILEGED=false
+DOCKER_BUILD_OUTPUT=${DOCKER_BUILDKIT_SETTING:-true}
DOCKER_CLEANUP_CMD=false
DOCKER_MEMORY="4g"
DOCKER_PLATFORM=""
@@ -67,6 +68,7 @@ function docker_usage
yetus_add_option "--docker-buildkit=<bool>" "Set the Docker BuildKit
availability (default: ${DOCKER_BUILDKIT_SETTING})'"
if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
yetus_add_option "--docker-bash-debug=<bool>" "Enable bash -x mode running
in a container (default: ${YETUS_DOCKER_BASH_DEBUG})"
+ yetus_add_option "--docker-build-output=<bool>" "Send docker build output
to the screen (default: '${DOCKER_BUILD_OUTPUT}')"
yetus_add_option "--docker-cache-from=<image>" "Comma delimited images to
use as a cache when building"
yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the
base (default: '${DOCKERFILE_DEFAULT}')"
yetus_add_option "--dockerind=<bool>" "Enable Docker-in-Docker by mounting
the Docker socket in the container (default: '${DOCKER_IN_DOCKER}')"
@@ -106,6 +108,10 @@ function docker_parse_args
delete_parameter "${i}"
DOCKER_BUILDKIT_SETTING=${i#*=}
;;
+ --docker-build-output=*)
+ delete_parameter "${i}"
+ DOCKER_BUILD_OUTPUT=${i#*=}
+ ;;
--docker-cache-from=*)
delete_parameter "${i}"
DOCKER_CACHE_FROM=${i#*=}
@@ -734,20 +740,37 @@ function docker_run_image
cachefrom=("--cache-from=yetus/${PROJECT_NAME}:${gitfilerev},${DOCKER_CACHE_FROM}")
fi
- if ! dockercmd build \
- "${dockplat[@]}" \
- "${cachefrom[@]}" \
- --label org.apache.yetus=\"\" \
- --label org.apache.yetus.testpatch.project="${PROJECT_NAME}" \
- --tag "${baseimagename}" \
- "${DOCKER_EXTRABUILDARGS[@]}" \
- -f "${buildfile}" \
- "${dockerdir}"; then
+ dockerbuild=(build "${dockplat[@]}" "${cachefrom[@]}")
+ dockerbuild+=(--label org.apache.yetus=\"\")
+ dockerbuild+=(--label org.apache.yetus.testpatch.project="${PROJECT_NAME}")
+ dockerbuild+=(--tag "${baseimagename}")
+ dockerbuild+=("${DOCKER_EXTRABUILDARGS[@]}")
+ dockerbuild+=(-f "${buildfile}")
+ dockerbuild+=("${dockerdir}")
+
+ if [[ "${DOCKER_BUILD_OUTPUT}" == 'true' ]]; then
+ echo "Starting docker build..."
+ dockercmd "${dockerbuild[@]}"
+ dockerlogfile=""
+ else
+ echo "Starting docker build..."
+ dockercmd "${dockerbuild[@]}" > "${PATCH_DIR}/docker-build.log" 2>&1
+ dockerlogfile="${PATCH_DIR}/docker-build.log"
+ fi
+
+ # shellcheck disable=SC2181
+ if [[ $? != 0 ]]; then
popd >/dev/null || return 1
yetus_error "ERROR: Docker failed to build ${baseimagename}."
- add_vote_table_v2 -1 docker "" "Docker failed to build ${baseimagename}."
+ gitdockerfile=$(yetus_relative_dir "${BASEDIR}" "${DOCKERFILE}")
+ echo "${gitdockerfile}:1:Failed to build Dockerfile" >
"${PATCH_DIR}/dockerline.log"
+ bugsystem_linecomments_queue docker "${PATCH_DIR}/dockerline.log"
+ rm "${PATCH_DIR}/dockerline.log"
+ add_vote_table_v2 -1 docker "${dockerlogfile}" "Docker failed to build
${baseimagename}."
bugsystem_finalreport 1
cleanup_and_exit 1
+ elif [[ -n "${dockerlogfile}" ]]; then
+ add_vote_table_v2 +1 docker "${dockerlogfile}" "Docker build log
${baseimagename}."
fi
popd >/dev/null || return 1
fi
diff --git a/precommit/src/main/shell/robots.d/travisci.sh
b/precommit/src/main/shell/robots.d/travisci.sh
index 05ad34b..c3b778d 100755
--- a/precommit/src/main/shell/robots.d/travisci.sh
+++ b/precommit/src/main/shell/robots.d/travisci.sh
@@ -26,6 +26,11 @@ if [[ "${TRAVIS}" == true ]] &&
DOCKER_BUILDKIT_SETTING=false
unset DOCKER_BUILDKIT
+ # Travis has massive limits on log output that it
+ # captures so stop this one for sure
+ DOCKER_BUILD_OUTPUT=false
+ export DOCKER_BUILD_OUTPUT
+
# shellcheck disable=SC2034
if [[ -n "${ARTIFACTS_PATH}" ]]; then
PATCH_DIR=${ARTIFACTS_PATH%%:*}
