This is an automated email from the ASF dual-hosted git repository. chesnay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-docker.git
commit 8513a68143c61f10114fc94d09921a1193741dd7 Author: Chesnay Schepler <ches...@apache.org> AuthorDate: Thu Apr 30 10:11:18 2020 +0200 [FLINK-17439] Introduce development branches - remove generates/templates; these now reside in the development branches - remove testing files; these now reside in the development branches - update README.md to reflect new development/release workflow --- .travis.yml | 9 -- Dockerfile-debian.template | 87 --------------- README.md | 59 +++++++--- add-custom.sh | 49 --------- add-version.sh | 113 ------------------- common.sh | 26 ----- docker-entrypoint.sh | 114 ------------------- testing/run_travis_tests.sh | 31 ------ testing/testing_lib.sh | 261 -------------------------------------------- 9 files changed, 43 insertions(+), 706 deletions(-) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a4e4f1d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -os: linux -dist: bionic -language: shell - -services: -- docker - -script: -- testing/run_travis_tests.sh diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template deleted file mode 100644 index 7222314..0000000 --- a/Dockerfile-debian.template +++ /dev/null @@ -1,87 +0,0 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -FROM openjdk:8-jre - -# Install dependencies -RUN set -ex; \ - apt-get update; \ - apt-get -y install libsnappy1v5 gettext-base; \ - rm -rf /var/lib/apt/lists/* - -# Grab gosu for easy step-down from root -ENV GOSU_VERSION 1.11 -RUN set -ex; \ - wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ - wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - for server in ha.pool.sks-keyservers.net $(shuf -e \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ - done && \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - chmod +x /usr/local/bin/gosu; \ - gosu nobody true - -# Configure Flink version -ENV FLINK_TGZ_URL=%%BINARY_DOWNLOAD_URL%% \ - FLINK_ASC_URL=%%ASC_DOWNLOAD_URL%% \ - GPG_KEY=%%GPG_KEY%% \ - CHECK_GPG=%%CHECK_GPG%% - -# Prepare environment -ENV FLINK_HOME=/opt/flink -ENV PATH=$FLINK_HOME/bin:$PATH -RUN groupadd --system --gid=9999 flink && \ - useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink -WORKDIR $FLINK_HOME - -# Install Flink -RUN set -ex; \ - wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ - \ - if [ "$CHECK_GPG" = "true" ]; then \ - wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ - export GNUPGHOME="$(mktemp -d)"; \ - for server in ha.pool.sks-keyservers.net $(shuf -e \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ - done && \ - gpg --batch --verify flink.tgz.asc flink.tgz; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" flink.tgz.asc; \ - fi; \ - \ - tar -xf flink.tgz --strip-components=1; \ - rm flink.tgz; \ - \ - chown -R flink:flink .; - -# Configure container -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] -EXPOSE 6123 8081 -CMD ["help"] diff --git a/README.md b/README.md index ff13dd7..ef2b814 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,24 @@ Flink Docker image lifecycle https://github.com/docker-library/official-images/blob/master/library/flink). +Development workflow +---------------------------- + +The `master` branch of this repository serves as a pure publishing area for releases. + +Development happens on the various `dev-X` branches. + +Pull requests for a specific version should be opened against the respective `dev-<version>` branch. +Pull requests for all versions, or for the next minor Flink release, should be opened against the `dev-master` branch. + +### CI + +The `dev-master` branch is tested against nightly Flink snapshots for the next minor Flink version. This allows us to +develop features in tandem with Flink. + +The `dev-1.x` branches are tested against the latest corresponding patch Flink release, to ensure any changes we make +are compatible with the currently used Flink version. + Workflow for new Flink releases ------------------------------- @@ -39,24 +57,30 @@ There are additional steps required when a new Flink minor version (x.y.0) is re ### Release workflow -When a new release of Flink is available, the Dockerfiles in this repo should be updated and a new +When a new release of Flink is available, the Dockerfiles in the `master` branch should be updated and a new manifest sent to the Docker Library [`official-images`]( https://github.com/docker-library/official-images) repo. -Updating the Dockerfiles involves 3 steps: - -1. Update `add-version.sh` with the GPG key ID of the key used to sign the new release - * Commit this change with message `Add GPG key for x.y.z release` <sup>\[[example]( - https://github.com/apache/flink-docker/commit/94845f46c0f0f2de80d4a5ce309db49aff4655d0)]</sup> -2. Remove any existing Dockerfiles from the same minor version - * e.g. `rm -r 1.2`, if the new Flink version is `1.2.1` -3. Run `add-version.sh` with the appropriate arguments (`-r flink-minor-version -f - flink-full-version`) - * e.g. `./add-version.sh -r 1.2 -f 1.2.1` - * Commit these changes with message `Update Dockerfiles for x.y.z release` <sup>\[[example]( +The Dockerfiles are generated on the respective `dev-<version>` branches, and copied over to the `master` branch for +publishing. + +Updating the Dockerfiles involves the following steps: + +1. Generate the Dockerfiles + * Checkout the `dev-x.y` branch of the respective release, e.g., dev-1.9 + * Update `add-version.sh` with the GPG key ID of the key used to sign the new release + * Commit this change with message `Add GPG key for x.y.z release` <sup>\[[example]( + https://github.com/apache/flink-docker/commit/94845f46c0f0f2de80d4a5ce309db49aff4655d0)]</sup> + * Create a pull request against the `dev-x.y` branch containing this commit. + * Run `add-version.sh` with the appropriate arguments (`-r flink-minor-version -f flink-full-version`) + * e.g. `./add-version.sh -r 1.2 -f 1.2.1` +2. Update Dockerfiles on the the `master` branch + * Remove any existing Dockerfiles from the same minor version + * e.g. `rm -r 1.2`, if the new Flink version is `1.2.1` + * Copy the generated Dockerfiles from the `dev-x.y` branch to `master` + * Commit the changes with message `Update Dockerfiles for x.y.z release` <sup>\[[example]( https://github.com/apache/flink-docker/commit/5920fd775ca1a8d03ee959d79bceeb5d6e8f35a1)]</sup> - -A pull request can then be opened on this repo with the changes. + * Create a pull request against the `master` branch containing this commit. Once the pull request has been merged, a new manifest should be generated and a pull request opened on the Docker Library [`official-images`](https://github.com/docker-library/official-images) repo. @@ -79,14 +103,17 @@ available shortly thereafter. ### Release checklist +Checklist for the `dev` branch: - [ ] The GPG key ID of the key used to sign the release has been added to `add-version.sh` and committed with the message `Add GPG key for x.y.z release` +- [ ] `./add-version.sh -r x.y -f x.y.z` has been run on the respective dev branch + +Checklist for the `master` branch: - [ ] _(new minor releases only)_ any unsupported Flink minor version Dockerfiles have been removed (only two `x.y/` directories should be present) - [ ] _(new patch releases only)_ any existing generated files for the same minor version have been removed -- [ ] `./add-version.sh -r x.y -f x.y.z` has been run, and the updated Dockerfiles committed with - the message `Update Dockerfiles for x.y.z release` +- [ ] The updated Dockerfiles have been committed with the message `Update Dockerfiles for x.y.z release` - [ ] _(new minor releases only)_ the `aliases` array in `generate-stackbrew-library.sh` has been updated with `[x.y]='latest'` and committed with the message `Update latest image tag to x.y` - [ ] A pull request with the above changes has been opened on this repo and merged diff --git a/add-custom.sh b/add-custom.sh deleted file mode 100755 index c863f3f..0000000 --- a/add-custom.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -# Use this script to build the Dockerfiles against an arbitrary -# Flink distribution. -# This is exlusively for development purposes. - -source "$(dirname "$0")"/common.sh - -function usage() { - echo >&2 "usage: $0 -u binary-download-url [-n name]" -} - -binary_download_url= -name=custom - -while getopts u:n:h arg; do - case "$arg" in - u) - binary_download_url=$OPTARG - ;; - n) - name=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - usage - exit 1 - ;; - esac -done - -if [ -z "${binary_download_url}" ]; then - usage - exit 1 -fi - -mkdir -p "dev" - -echo -n >&2 "Generating Dockerfiles..." -for source_variant in "${SOURCE_VARIANTS[@]}"; do - dir="dev/${name}-${source_variant}" - rm -rf "${dir}" - - generate "${dir}" "${binary_download_url}" "" "" false ${source_variant} -done -echo >&2 " done." diff --git a/add-version.sh b/add-version.sh deleted file mode 100755 index 329ed70..0000000 --- a/add-version.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -e - -# Use this script to rebuild the Dockerfiles and all variants for a particular -# release. Before running this, you must first delete the existing release -# directory. -# -# TODO: to conform with other similar setups, this likely needs to become -# "update.sh" and be taught how to derive the latest version (e.g. 1.2.0) from -# a given release (e.g. 1.2) and assemble a .travis.yml file dynamically. -# -# See other repos (e.g. httpd, cassandra) for update.sh examples. - -source "$(dirname "$0")"/common.sh - -function usage() { - echo >&2 "usage: $0 -r flink-release -f flink-version" -} - -function error() { - local msg="$1" - if [ -n "$2" ]; then - local code="$2" - else - local code=1 - fi - echo >&2 "$msg" - exit "$code" -} - -flink_release= # Like 1.2 -flink_version= # Like 1.2.0 - -while getopts r:f:h arg; do - case "$arg" in - r) - flink_release=$OPTARG - ;; - f) - flink_version=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - usage - exit 1 - ;; - esac -done - -if [ -z "$flink_release" ] || [ -z "$flink_version" ]; then - usage - exit 1 -fi - -if [[ ! "$flink_version" =~ ^$flink_release\.+ ]]; then - error "Flink release must be prefix of version" -fi - -# Defaults, can vary between versions -scala_versions=( 2.11 2.12 ) -gpg_key= - -# Version-specific variants (example) -# if [ "$flink_release" = "x.y" ]; then -# scala_versions=( 2.10 2.11 2.12 ) -# fi - -# No real need to cull old versions -if [ "$flink_version" = "1.8.0" ]; then - gpg_key="F2A67A8047499BBB3908D17AA8F4FD97121D7293" -elif [ "$flink_version" = "1.8.1" ]; then - gpg_key="8FEA1EE9D0048C0CCC70B7573211B0703B79EA0E" -elif [ "$flink_version" = "1.8.2" ]; then - gpg_key="E2C45417BED5C104154F341085BACB5AEFAE3202" -elif [ "$flink_version" = "1.8.3" ]; then - gpg_key="EF88474C564C7A608A822EEC3FF96A2057B6476C" -elif [ "$flink_version" = "1.9.0" ]; then - gpg_key="1C1E2394D3194E1944613488F320986D35C33D6A" -elif [ "$flink_version" = "1.9.1" ]; then - gpg_key="E2C45417BED5C104154F341085BACB5AEFAE3202" -elif [ "$flink_version" = "1.9.2" ]; then - gpg_key="EF88474C564C7A608A822EEC3FF96A2057B6476C" -elif [ "$flink_version" = "1.9.3" ]; then - gpg_key="6B6291A8502BA8F0913AE04DDEB95B05BF075300" -elif [ "$flink_version" = "1.10.0" ]; then - gpg_key="BB137807CEFBE7DD2616556710B12A1F89C115E8" -else - error "Missing GPG key ID for this release" -fi - -if [ -d "$flink_release" ]; then - error "Directory $flink_release already exists; delete before continuing" -fi - -mkdir "$flink_release" - -echo -n >&2 "Generating Dockerfiles..." -for source_variant in "${SOURCE_VARIANTS[@]}"; do - for scala_version in "${scala_versions[@]}"; do - dir="$flink_release/scala_${scala_version}-${source_variant}" - - flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz - - flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}" - # Not all mirrors have the .asc files - flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc - - generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${source_variant} - done -done -echo >&2 " done." diff --git a/common.sh b/common.sh deleted file mode 100644 index 0232930..0000000 --- a/common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -e - -# Defaults, can vary between versions -export SOURCE_VARIANTS=( debian ) - -function generate() { - dir=$1 - binary_download_url=$2 - asc_download_url=$3 - gpg_key=$4 - check_gpg=$5 - source_variant=$6 - - mkdir "$dir" - cp docker-entrypoint.sh "$dir/docker-entrypoint.sh" - - # '&' has special semantics in sed replacement patterns - escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/') - - sed \ - -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \ - -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \ - -e "s/%%GPG_KEY%%/$gpg_key/" \ - -e "s/%%CHECK_GPG%%/${check_gpg}/" \ - "Dockerfile-$source_variant.template" > "$dir/Dockerfile" -} diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 011ccea..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/sh - -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - -# If unspecified, the hostname of the container is taken as the JobManager address -JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)} -CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml" - -drop_privs_cmd() { - if [ $(id -u) != 0 ]; then - # Don't need to drop privs if EUID != 0 - return - elif [ -x /sbin/su-exec ]; then - # Alpine - echo su-exec flink - else - # Others - echo gosu flink - fi -} - -copy_plugins_if_required() { - if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then - return 0 - fi - - echo "Enabling required built-in plugins" - for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do - echo "Linking ${target_plugin} to plugin directory" - plugin_name=${target_plugin%.jar} - - mkdir -p "${FLINK_HOME}/plugins/${plugin_name}" - if [ ! -e "${FLINK_HOME}/opt/${target_plugin}" ]; then - echo "Plugin ${target_plugin} does not exist. Exiting." - exit 1 - else - ln -fs "${FLINK_HOME}/opt/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}" - echo "Successfully enabled ${target_plugin}" - fi - done -} - -set_config_option() { - local option=$1 - local value=$2 - - # escape periods for usage in regular expressions - local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g") - - # either override an existing entry, or append a new one - if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then - sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}" - else - echo "${option}: ${value}" >> "${CONF_FILE}" - fi -} - -set_common_options() { - set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS} - set_config_option blob.server.port 6124 - set_config_option query.server.port 6125 -} - -if [ "$1" = "help" ]; then - echo "Usage: $(basename "$0") (jobmanager|taskmanager|help)" - exit 0 -elif [ "$1" = "jobmanager" ]; then - shift 1 - echo "Starting Job Manager" - copy_plugins_if_required - - set_common_options - - if [ -n "${FLINK_PROPERTIES}" ]; then - echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" - fi - envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" - - exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@" -elif [ "$1" = "taskmanager" ]; then - shift 1 - echo "Starting Task Manager" - copy_plugins_if_required - - TASK_MANAGER_NUMBER_OF_TASK_SLOTS=${TASK_MANAGER_NUMBER_OF_TASK_SLOTS:-$(grep -c ^processor /proc/cpuinfo)} - - set_common_options - set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS} - - if [ -n "${FLINK_PROPERTIES}" ]; then - echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" - fi - envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" - - exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@" -fi - -exec "$@" diff --git a/testing/run_travis_tests.sh b/testing/run_travis_tests.sh deleted file mode 100755 index f00890e..0000000 --- a/testing/run_travis_tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -e - -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) - -. "${SCRIPT_DIR}/testing_lib.sh" - -IS_PULL_REQUEST= -if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - IS_PULL_REQUEST=1 -fi - -BRANCH="$TRAVIS_BRANCH" - -if [ -n "$IS_PULL_REQUEST" ]; then - changed_files="$(git diff --name-only $BRANCH...HEAD)" - - echo "Changed files in this pull request:" - echo "${changed_files}" - echo -fi - -if [ -z "$IS_PULL_REQUEST" ] && [ "$BRANCH" = "master" ]; then - # Test all images on master - smoke_test_all_images - smoke_test_non_root -else - # For pull requests and branches, test one image - smoke_test_one_image -fi - -# vim: et ts=2 sw=2 diff --git a/testing/testing_lib.sh b/testing/testing_lib.sh deleted file mode 100644 index 36dfe32..0000000 --- a/testing/testing_lib.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/bin/bash -e - -# This test script discovers all image variants, builds them, then runs a -# simple smoke test for each: -# - start up a jobmanager -# - wait for the /overview web UI endpoint to return successfully -# - start up a taskmanager -# - wait for the /overview web UI endpoint to report 1 connected taskmanager - -CURL_TIMEOUT=1 -CURL_COOLDOWN=1 -CURL_MAX_TRIES=30 -CURL_ENDPOINT=localhost:8081/overview -IMAGE_REPO=docker-flink-test -NETWORK_NAME=docker-flink-test-net - -function image_tag() { - local dockerfile - dockerfile="$1" - - local variant minor_version - variant="$(basename "$(dirname "$dockerfile")")" - minor_version="$(basename "$(dirname "$(dirname "$dockerfile")")")" - - echo "${minor_version}-${variant}" -} - -function image_name() { - local image_tag - image_tag="$1" - - echo "${IMAGE_REPO}:${image_tag}" -} - -function build_image() { - local dockerfile - dockerfile="$1" - - local image_tag image_name dockerfile_dir - image_tag="$(image_tag "$dockerfile")" - image_name="$(image_name "$image_tag")" - dockerfile_dir="$(dirname "$dockerfile")" - - echo >&2 "===> Building ${image_tag} image..." - docker build -t "$image_name" "$dockerfile_dir" -} - -function internal_run() { - local dockerfile="$1" - local docker_run_command="$2" - local args="$3" - - local image_tag image_name - image_tag="$(image_tag "$dockerfile")" - image_name="$(image_name "$image_tag")" - - echo >&2 "===> Starting ${image_tag} ${args}..." - - eval "docker run --rm --detach --network $NETWORK_NAME -e JOB_MANAGER_RPC_ADDRESS=jobmanager ${docker_run_command} $image_name ${args}" -} - -function internal_run_jobmanager() { - internal_run "$1" "--name jobmanager --publish 6123:6123 --publish 8081:8081 $2" jobmanager -} - -function run_jobmanager() { - internal_run_jobmanager "$1" "" -} - -function run_jobmanager_non_root() { - internal_run_jobmanager "$1" "--user flink" -} - -function internal_run_taskmanager() { - internal_run "$1" "--name taskmanager $2" "taskmanager" -} - -function run_taskmanager() { - internal_run_taskmanager "$1" "" -} - -function run_taskmanager_non_root() { - internal_run_taskmanager "$1" "--user flink" -} - -function wait_for_jobmanager() { - local dockerfile - dockerfile="$1" - - local image_tag - image_tag="$(image_tag "$dockerfile")" - - i=0 - echo >&2 "===> Waiting for ${image_tag} jobmanager to be ready..." - while true; do - i=$((i+1)) - - set +e - - curl \ - --silent \ - --max-time "$CURL_TIMEOUT" \ - "$CURL_ENDPOINT" \ - > /dev/null - - result=$? - - set -e - - if [ "$result" -eq 0 ]; then - break - fi - - if [ "$i" -gt "$CURL_MAX_TRIES" ]; then - echo >&2 "===> \$CURL_MAX_TRIES exceeded waiting for jobmanager to be ready" - return 1 - fi - - sleep "$CURL_COOLDOWN" - done - - echo >&2 "===> ${image_tag} jobmanager is ready." -} - -function test_image() { - local dockerfile - dockerfile="$1" - - local image_tag - image_tag="$(image_tag "$dockerfile")" - - i=0 - echo >&2 "===> Waiting for ${image_tag} taskmanager to connect..." - while true; do - i=$((i+1)) - - set +e - - local overview - overview="$(curl \ - --silent \ - --max-time "$CURL_TIMEOUT" \ - "$CURL_ENDPOINT")" - - num_taskmanagers="$(echo "$overview" | jq .taskmanagers)" - - set -e - - if [ "$num_taskmanagers" = "1" ]; then - break - fi - - if [ "$i" -gt "$CURL_MAX_TRIES" ]; then - echo >&2 "===> \$CURL_MAX_TRIES exceeded for taskmanager to connect" - return 1 - fi - - sleep "$CURL_COOLDOWN" - done - echo >&2 "===> ${image_tag} taskmanager connected." -} - -function create_network() { - docker network create "$NETWORK_NAME" > /dev/null -} - -# Find and kill any remaining containers attached to the network, then remove -# the network and any images produced by the build. -function cleanup() { - local containers - containers="$(docker ps --quiet --filter network="$NETWORK_NAME")" - - if [ -n "$containers" ]; then - echo >&2 -n "==> Killing $(echo -n "$containers" | grep -c '^') orphaned container(s)..." - echo "$containers" | xargs docker kill > /dev/null - echo >&2 " done." - fi - - docker network rm "$NETWORK_NAME" > /dev/null 2>&1 || true - - local images - images="$(docker images --quiet --filter reference="$IMAGE_REPO")" - - if [ -n "$images" ]; then - echo >&2 -n "==> Deleting $(echo -n "$images" | grep -c '^') image(s)..." - echo "$images" | xargs docker rmi > /dev/null - echo >&2 " done." - fi -} - -# For each image, run a jobmanager and taskmanager and verify they start up and connect to each -# other successfully. -function smoke_test_all_images() { - create_network - trap cleanup EXIT RETURN - - local jobmanager_container_id - local taskmanager_container_id - local dockerfiles - dockerfiles="$(ls ./*/*/Dockerfile)" - - echo >&2 "==> Test all images" - - for dockerfile in $dockerfiles; do - build_image "$dockerfile" - jobmanager_container_id="$(run_jobmanager "$dockerfile")" - taskmanager_container_id="$(run_taskmanager "$dockerfile")" - wait_for_jobmanager "$dockerfile" - test_image "$dockerfile" - docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null - done -} - -# Same as smoke_test_all_images, but test only the last image alphabetically (presumed to be the -# most recent). -function smoke_test_one_image() { - create_network - trap cleanup EXIT RETURN - - local jobmanager_container_id - local taskmanager_container_id - local dockerfiles - dockerfiles="$dockerfiles $(ls ./*/*/Dockerfile | tail -n 1)" - - echo >&2 "==> Test one image" - - for dockerfile in $dockerfiles; do - build_image "$dockerfile" - jobmanager_container_id="$(run_jobmanager "$dockerfile")" - taskmanager_container_id="$(run_taskmanager "$dockerfile")" - wait_for_jobmanager "$dockerfile" - test_image "$dockerfile" - docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null - done -} - -# Similar to smoke_test_one_image, but test one debian image and one alpine image running as a -# non-root user. -function smoke_test_non_root() { - create_network - trap cleanup EXIT RETURN - - local jobmanager_container_id - local taskmanager_container_id - local dockerfiles - dockerfiles="$dockerfiles $(ls ./*/*-debian/Dockerfile | tail -n 1)" - dockerfiles="$dockerfiles $(ls ./*/*-alpine/Dockerfile | tail -n 1)" - - echo >&2 "==> Test images running as non-root" - - for dockerfile in $dockerfiles; do - build_image "$dockerfile" - jobmanager_container_id="$(run_jobmanager_non_root "$dockerfile")" - taskmanager_container_id="$(run_taskmanager_non_root "$dockerfile")" - wait_for_jobmanager "$dockerfile" - test_image "$dockerfile" - docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null - done -} - -# vim: ts=4 sw=4 et