This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 5305b5b YETUS-1054. Support BuildKit (#190)
5305b5b is described below
commit 5305b5b43687ae34a14986db2dd3a0fd43a9601d
Author: Allen Wittenauer <[email protected]>
AuthorDate: Mon Nov 2 10:05:15 2020 -0800
YETUS-1054. Support BuildKit (#190)
---
.../in-progress/precommit/docker.html.md | 6 +
.../in-progress/precommit/robots/travisci.html.md | 6 +-
hooks/build | 6 +-
precommit/src/main/shell/core.d/docker.sh | 11 +
precommit/src/main/shell/robots.d/travisci.sh | 5 +
.../src/main/shell/test-patch-docker/Dockerfile | 244 +++++++++++++--------
start-build-env.sh | 4 +
7 files changed, 192 insertions(+), 90 deletions(-)
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
b/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
index 30b454d..24ca7f8 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/docker.html.md
@@ -29,6 +29,7 @@
* [Using a cache](#using-a-cache)
* [Platforms](#platforms)
* [Container Directory](#container-directory)
+* [BuildKit](#buildkit)
* [Resource Controls](#resource-controls)
* [Privileged Mode](#privileged-mode)
* [Docker in Docker](#docker-in-docker)
@@ -80,6 +81,11 @@ By default, precommit will use `/precommit` as the directory
where it will store
not provided by other flags in system (such as `--basedir` or `--patch-dir`).
If that directory conflicts with some other
need, then the `--docker-work-dir` option may be provided to set a different
path.
+# BuildKit
+
+By default, precommit will enable [Docker
BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/)
+unless told otherwise with `--docker-buildkit=false` or if the CI system has
known limitations.
+
# Resource Controls
Docker's `--memory` flag is supported via the `--dockermemlimit` option. This
enables the container's memory size to be limited. This may be important to
set to prevent things like broken unit tests bringing down the entire build
server. See [the Docker
documentation](https://docs.docker.com/engine/admin/resource_constraints/) for
more details. Apache Yetus also sets the `--oom-score-adj` to 500 in order to
offer itself as the first processes to be killed if memory is low.
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
index 82370f4..23fdead 100644
---
a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
+++
b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
@@ -27,7 +27,11 @@ If `${ARTIFACTS_PATH}` is configured, then `--patch-dir` is
set to the first lis
Personalities will override the auto-detected Github repository information.
It may be necessary to manually configure it in your `.travis.yml` file.
-As of this writing, it is not possible to make the Travis CI build environment
use the Apache Yetus pre-built docker images without using `docker run` in the
before_install phase. Therefore, using the image is the same as described in
the [Apache Yetus Docker Hub Images](../../../../../yetus-docker-image) page.
+As of this writing, it is not possible to make the Travis CI build environment
use the Apache Yetus pre-built docker
+images without using `docker run` in the before_install phase. Therefore,
using the image is the same as described
+in the [Apache Yetus Docker Hub Images](../../../../../yetus-docker-image)
page.
+
+ NOTE: As of this writing (2020-10-31), Travis CI does not support Docker
BuildKit. It is disabled by default.
See also:
diff --git a/hooks/build b/hooks/build
index 622589c..0db14e3 100755
--- a/hooks/build
+++ b/hooks/build
@@ -16,6 +16,10 @@
GIT_URL=$(git config --get remote.origin.url)
+# shellcheck disable=SC2034
+DOCKER_BUILDKIT=1
+export DOCKER_BUILDKIT
+
# Build the -base image
docker build \
--label "git-commit-sha=${SOURCE_COMMIT}" \
@@ -32,4 +36,4 @@ docker build \
--tag "${DOCKER_REPO}:${DOCKER_TAG}" \
--build-arg DOCKER_TAG="${DOCKER_TAG}" \
--build-arg DOCKER_REPO="${DOCKER_REPO}" \
- .
\ No newline at end of file
+ .
diff --git a/precommit/src/main/shell/core.d/docker.sh
b/precommit/src/main/shell/core.d/docker.sh
index 82dccff..7d44c0c 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -16,6 +16,8 @@
DOCKERMODE=false
DOCKERCMD=$(command -v docker 2>/dev/null)
+DOCKER_BUILDKIT_SETTING=${DOCKER_BUILDKIT_SETTING:-true}
+unset DOCKER_BUILDKIT
DOCKER_ID=${RANDOM}
DOCKER_DESTRUCTIVE=true
DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
@@ -62,6 +64,7 @@ function docker_usage
yetus_add_option "--docker" "Spawn a docker container"
fi
yetus_add_option "--dockercmd=<file>" "Command to use as docker executable
(default: '${DOCKERCMD}')"
+ yetus_add_option "--docker-buildkit=<bool>" "Set the Docker BuildKit
availability (default: ${DOCKER_BUILDKIT_SETTING})'"
if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
yetus_add_option "--docker-bash-debug=<bool>" "Enable bash -x mode running
in a container (default: ${YETUS_DOCKER_BASH_DEBUG})"
yetus_add_option "--docker-cache-from=<image>" "Comma delimited images to
use as a cache when building"
@@ -99,6 +102,10 @@ function docker_parse_args
YETUS_DOCKER_BASH_DEBUG=${i#*=}
add_docker_env YETUS_DOCKER_BASH_DEBUG
;;
+ --docker-buildkit=*)
+ delete_parameter "${i}"
+ DOCKER_BUILDKIT_SETTING=${i#*=}
+ ;;
--docker-cache-from=*)
delete_parameter "${i}"
DOCKER_CACHE_FROM=${i#*=}
@@ -639,6 +646,10 @@ function docker_run_image
big_console_header "Docker Image Creation"
start_clock
+ if [[ "${DOCKER_BUILDKIT_SETTING}" == true ]]; then
+ export DOCKER_BUILDKIT=1
+ fi
+
if [[ -n "${DOCKER_PLATFORM}" ]]; then
dockplat=('--platform' "${DOCKER_PLATFORM}")
fi
diff --git a/precommit/src/main/shell/robots.d/travisci.sh
b/precommit/src/main/shell/robots.d/travisci.sh
index 8898e3e..05ad34b 100755
--- a/precommit/src/main/shell/robots.d/travisci.sh
+++ b/precommit/src/main/shell/robots.d/travisci.sh
@@ -21,6 +21,11 @@ if [[ "${TRAVIS}" == true ]] &&
# shellcheck disable=SC2034
ROBOT=true
+ # Travis runs some ancient version of docker, so...
+ # shellcheck disable=SC2034
+ DOCKER_BUILDKIT_SETTING=false
+ unset DOCKER_BUILDKIT
+
# shellcheck disable=SC2034
if [[ -n "${ARTIFACTS_PATH}" ]]; then
PATCH_DIR=${ARTIFACTS_PATH%%:*}
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile
b/precommit/src/main/shell/test-patch-docker/Dockerfile
index d5873f3..d874af2 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -23,7 +23,7 @@
#
###############
-FROM ubuntu:focal
+FROM ubuntu:focal AS yetusbase
## NOTE to committers: if this gets moved from Xenial to something else, be
## sure to also fix the gpg link in asf-site-src as appropriate
@@ -68,6 +68,144 @@ ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
####
+# Install GNU C/C++ (everything generally needs this)
+####
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ g++ \
+ gcc \
+ libc-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+###
+# Install golang as part of base so we can do each
+# helper utility in parallel. go bins are typically
+# statically linked, so this is perfectly safe.
+###
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:longsleep/golang-backports \
+ && apt-get -q update \
+ && apt-get -q install --no-install-recommends -y golang-go \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+############
+# Fetch all of the non-conflicting bits in parallel
+#############
+
+####
+# Install Apache Creadur RAT jar
+####
+FROM yetusbase AS apacherat
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN mkdir -p /opt/apache-rat \
+ && curl -f -s -L -o /tmp/rat.tgz \
+
'https://www.apache.org/dyn/closer.cgi?action=download&filename=/creadur/apache-rat-0.13/apache-rat-0.13-bin.tar.gz'
\
+ && shasum -a 512 /tmp/rat.tgz \
+ | awk
'$1!="2c1e12eace7b80a9b6373c2f5080fbf63d3fa8d9248f3a17bd05de961cd3ca3c4549817b8b7320a84f0c323194edad0abdb86bdfec3976227a228e2143e14a54"
{exit(1)}' \
+ && tar --strip-components 1 -C /opt/apache-rat -xpzf /tmp/rat.tgz \
+ && rm /tmp/rat.tgz \
+ && mv /opt/apache-rat/apache-rat-0.13.jar /opt/apache-rat/apache-rat.jar
+
+#####
+# Install SpotBugs
+#####
+FROM yetusbase AS spotbugs
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL
https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/4.1.2/spotbugs-4.1.2.tgz
-o spotbugs.tgz \
+ && curl -sSL
https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/4.1.2/spotbugs-4.1.2.tgz.sha1
-o spotbugs.tgz.sha1 \
+ && echo -n " spotbugs.tgz" >> spotbugs.tgz.sha1 \
+ && shasum -c spotbugs.tgz.sha1 \
+ && mkdir -p /opt/spotbugs \
+ && tar -C /opt/spotbugs --strip-components 1 -xpf spotbugs.tgz \
+ && rm spotbugs.tgz spotbugs.tgz.sha1 \
+ && chmod a+rx /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+####
+# Install shellcheck (shell script lint)
+####
+FROM yetusbase AS shellcheck
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+
https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
\
+ | tar --strip-components 1 --wildcards -xJf - '*/shellcheck' \
+ && chmod a+rx shellcheck \
+ && mv shellcheck /bin/shellcheck \
+ && shasum -a 512 /bin/shellcheck \
+ | awk
'$1!="aae813283d49f18f95a205dca1c5184267d07534a08abc952ebea1958fee06f8a0207373b6770a083079ba875458ea9da443f2b9910a50dcd93b935048bc14f5"
{exit(1)}'
+
+####
+# Install hadolint (dockerfile lint)
+####
+FROM yetusbase AS hadolint
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+
https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-Linux-x86_64
\
+ -o /bin/hadolint \
+ && chmod a+rx /bin/hadolint \
+ && shasum -a 512 /bin/hadolint \
+ | awk
'$1!="df27253d374c143a606483b07a26234ac7b4bca40b4eba53e79609c81aa70146e7d5c145f90dcec71d6d1aad1048b7d9d2de68d92284f48a735d04d19c5c5559"
{exit(1)}'
+
+####
+# Install buf (protobuf lint)
+####
+FROM yetusbase AS buf
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+
https://github.com/bufbuild/buf/releases/download/v0.21.0/buf-Linux-x86_64.tar.gz
\
+ -o buf.tar.gz \
+ && shasum -a 256 buf.tar.gz \
+ | awk
'$1!="95aba62ac0ecc5a9120cc58c65cdcc85038633a816bddfe8398c5ae3b32803f1"
{exit(1)}' \
+ && tar -xzf buf.tar.gz -C /usr/local --strip-components 1 \
+ && rm buf.tar.gz
+
+####
+# Install bats (TAP-capable unit testing for shell scripts)
+####
+FROM yetusbase AS bats
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN git clone --branch v1.2.1 \
+ https://github.com/bats-core/bats-core.git \
+ /tmp/bats-core \
+ && /tmp/bats-core/install.sh /opt/bats \
+ && rm -rf /tmp/bats-core
+
+
+####
+# revive (golint on steroids)
+####
+FROM yetusbase AS revive
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN go get -u github.com/mgechev/revive
+
+
+####
+# checkmake (Makefile linter)
+####
+FROM yetusbase AS checkmake
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN go get -u github.com/mrtazz/checkmake
+
+####
+# golangci-lint (Multi-tool golang linter)
+####
+FROM yetusbase as golangci
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN (GO111MODULE=on go get
github.com/golangci/golangci-lint/cmd/[email protected])
+
+########
+#
+#
+# Content that needs to be installed in order due to packages...
+#
+#
+########
+
+FROM yetusbase
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+####
# Install java (first, since we want to dicate what form of Java)
####
@@ -91,7 +229,6 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y default-j
&& ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-
####
# Install ant
####
@@ -100,18 +237,6 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y ant \
&& rm -rf /var/lib/apt/lists/*
####
-# Install Apache Creadur RAT jar
-####
-RUN mkdir -p /opt/apache-rat \
- && curl -f -s -L -o /tmp/rat.tgz \
-
'https://www.apache.org/dyn/closer.cgi?action=download&filename=/creadur/apache-rat-0.13/apache-rat-0.13-bin.tar.gz'
\
- && shasum -a 512 /tmp/rat.tgz \
- | awk
'$1!="2c1e12eace7b80a9b6373c2f5080fbf63d3fa8d9248f3a17bd05de961cd3ca3c4549817b8b7320a84f0c323194edad0abdb86bdfec3976227a228e2143e14a54"
{exit(1)}' \
- && tar --strip-components 1 -C /opt/apache-rat -xpzf /tmp/rat.tgz \
- && rm /tmp/rat.tgz \
- && mv /opt/apache-rat/apache-rat-0.13.jar /opt/apache-rat/apache-rat.jar
-
-####
# Install GNU automake, GNU make, and related
####
RUN apt-get -q update && apt-get -q install --no-install-recommends -y
autoconf automake libtool make \
@@ -119,15 +244,6 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y autoconf
&& rm -rf /var/lib/apt/lists/*
####
-# Install bats (TAP-capable unit testing for shell scripts)
-####
-RUN git clone --branch v1.2.1 \
- https://github.com/bats-core/bats-core.git \
- /tmp/bats-core \
- && /tmp/bats-core/install.sh /usr/local \
- && rm -rf /tmp/bats-core
-
-####
# Install cmake
####
RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \
@@ -154,28 +270,7 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y findbugs
&& rm -rf /var/lib/apt/lists/*
ENV FINDBUGS_HOME /usr
-#####
-# Install SpotBugs
-#####
-RUN curl -sSL
https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/4.1.2/spotbugs-4.1.2.tgz
-o spotbugs.tgz \
- && curl -sSL
https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/4.1.2/spotbugs-4.1.2.tgz.sha1
-o spotbugs.tgz.sha1 \
- && echo -n " spotbugs.tgz" >> spotbugs.tgz.sha1 \
- && shasum -c spotbugs.tgz.sha1 \
- && mkdir -p /opt/spotbugs \
- && tar -C /opt/spotbugs --strip-components 1 -xpf spotbugs.tgz \
- && rm spotbugs.tgz spotbugs.tgz.sha1 \
- && chmod a+rx /opt/spotbugs/bin/*
-ENV SPOTBUGS_HOME /opt/spotbugs
-####
-# Install GNU C/C++
-####
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- g++ \
- gcc \
- libc-dev \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
######
# Install maven
@@ -283,26 +378,6 @@ RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc \
ENV PATH ${PATH}:/var/tmp/.bundler-gems/bin
ENV BUNDLE_PATH /var/tmp/.bundler-gems
-####
-# Install shellcheck (shell script lint)
-####
-RUN curl -sSL \
-
https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
\
- | tar --strip-components 1 --wildcards -xJf - '*/shellcheck' \
- && chmod a+rx shellcheck \
- && mv shellcheck /bin/shellcheck \
- && shasum -a 512 /bin/shellcheck \
- | awk
'$1!="aae813283d49f18f95a205dca1c5184267d07534a08abc952ebea1958fee06f8a0207373b6770a083079ba875458ea9da443f2b9910a50dcd93b935048bc14f5"
{exit(1)}'
-
-###
-# Install hadolint
-####
-RUN curl -sSL \
-
https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-Linux-x86_64
\
- -o /bin/hadolint \
- && chmod a+rx /bin/hadolint \
- && shasum -a 512 /bin/hadolint \
- | awk
'$1!="df27253d374c143a606483b07a26234ac7b4bca40b4eba53e79609c81aa70146e7d5c145f90dcec71d6d1aad1048b7d9d2de68d92284f48a735d04d19c5c5559"
{exit(1)}'
###
# Install npm and JSHint
@@ -315,32 +390,25 @@ RUN curl -sSL https://deb.nodesource.com/setup_14.x |
bash - \
&& npm install -g [email protected] [email protected] \
&& rm -rf /root/.npm
-###
-# Install golang and supported helpers
-###
-# hadolint ignore=DL3008
-RUN add-apt-repository -y ppa:longsleep/golang-backports \
- && apt-get -q update \
- && apt-get -q install --no-install-recommends -y golang-go \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/* \
- && go get -u github.com/mgechev/revive \
- && go get -u github.com/mrtazz/checkmake \
- && (GO111MODULE=on go get
github.com/golangci/golangci-lint/cmd/[email protected]) \
- && mv /root/go/bin/* /usr/local/bin \
- && rm -rf /root/go /root/.cache/go-build
+#####
+# Now all the stuff that was built in parallel
+#####
-RUN curl -sSL \
-
https://github.com/bufbuild/buf/releases/download/v0.21.0/buf-Linux-x86_64.tar.gz
\
- -o buf.tar.gz \
- && shasum -a 256 buf.tar.gz \
- | awk
'$1!="95aba62ac0ecc5a9120cc58c65cdcc85038633a816bddfe8398c5ae3b32803f1"
{exit(1)}' \
- && tar -xzf buf.tar.gz -C /usr/local --strip-components 1 \
- && rm buf.tar.gz
+COPY --from=apacherat /opt/apache-rat /opt/apache-rat
+COPY --from=spotbugs /opt/spotbugs /opt/spotbugs
+COPY --from=shellcheck /bin/shellcheck /bin/shellcheck
+COPY --from=hadolint /bin/hadolint /bin/hadolint
+COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
+COPY --from=bats /opt/bats /opt/bats
+RUN ln -s /opt/bats/bin/bats /usr/local/bin/bats
+
+COPY --from=revive /root/go/bin/* /usr/local/bin
+COPY --from=checkmake /root/go/bin/* /usr/local/bin
+COPY --from=golangci /root/go/bin/* /usr/local/bin
####
# YETUS CUT HERE
-# Anything after the above line is ignored by Yetus, so could
-# include other requirements not needed by your development
-# (but not build) environment
+# Magic text above! Everything from here on is ignored
+# by Yetus, so could include anything not needed
+# by your testing environment
###
diff --git a/start-build-env.sh b/start-build-env.sh
index 8c192ca..3e5bea4 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -20,6 +20,10 @@ ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")"
>/dev/null && pwd -P)
YETUS_DOCKER_REPO=${YETUS_DOCKER_REPO:-apache/yetus}
+# shellcheck disable=SC2034
+DOCKER_BUILDKIT=1
+export DOCKER_BUILDKIT
+
# moving to the path of the Dockerfile reduces the context
cd "${ROOTDIR}/precommit/src/main/shell/test-patch-docker"
