This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a27701a396f803e6c5434f2c2062393efa55f62a
Author: Jarek Potiuk <jarek.pot...@polidea.com>
AuthorDate: Sun Nov 8 11:20:31 2020 +0100
Uses always the same Python base image as used for CI image (#12177)
When new Python version is released (bugfixes), we rebuild the CI image
and replace it with the new one, however releasing of the python
image and CI image is often hours or even days apart (we only
release the CI image when tests pass in master with the new python
image). We already use a better approach for Github - we simply
push the new python image to our registry together with the CI
image and the CI jobs are always pulling them from our registry
knowing that the two - python and CI image are in sync.
This PR introduces the same approach. We not only push CI image
but also the corresponding Python image to our registry. This has
no ill effect - DockerHub handles it automatically and reuses
the layers of the image directly from the Python one so it is
merely a label that is stored in our registry that points to the
exact Python image that was used by the last pushed CI image.
(cherry picked from commit 75bdfaeb9b2d7f47599e951ccaccc515a180ca19)
---
IMAGES.rst | 11 ++++++++
scripts/ci/libraries/_build_images.sh | 11 +++-----
scripts/ci/libraries/_push_pull_remove_images.sh | 35 +++---------------------
3 files changed, 19 insertions(+), 38 deletions(-)
diff --git a/IMAGES.rst b/IMAGES.rst
index 45aa09ab..8c913db 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -46,6 +46,17 @@ where:
* The ``-ci`` suffix is added for CI images
* The ``-manifest`` is added for manifest images (see below for explanation of
manifest images)
+We also store (to increase speed of local build/pulls) python images that were
used to build
+the CI images. Each CI image, when built uses current python version of the
base images. Those
+python images are regularly updated (with bugfixes/security fixes), so for
example python3.8 from
+last week might be a different image than python3.8 today. Therefore whenever
we push CI image
+to airflow repository, we also push the python image that was used to build it
this image is stored
+as ``apache/airflow:python-3.8-<BRANCH_OR_TAG>``.
+
+Since those are simply snapshots of the existing python images, DockerHub does
not create a separate
+copy of those images - all layers are mounted from the original python images
and those are merely
+labels pointing to those.
+
Building docker images
======================
diff --git a/scripts/ci/libraries/_build_images.sh
b/scripts/ci/libraries/_build_images.sh
index cbd01c1..a2577cb 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -330,9 +330,11 @@ function build_images::get_docker_image_names() {
export
AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
# CI image to build
export
AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
-
# Default CI image
- export
AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci"
+ export
AIRFLOW_CI_PYTHON_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:python${PYTHON_MAJOR_MINOR_VERSION}-${BRANCH_NAME}"
+ # CI image to build
+ export
AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
+
# Base production image tag - used to build kubernetes tag as well
if [[ ${FORCE_AIRFLOW_PROD_BASE_TAG=} == "" ]]; then
@@ -374,11 +376,6 @@ function build_images::prepare_ci_build() {
export
GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}"
export
GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
fi
- if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" ==
"${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
- export DEFAULT_PROD_IMAGE="${AIRFLOW_CI_IMAGE_DEFAULT}"
- else
- export DEFAULT_PROD_IMAGE=""
- fi
export THE_IMAGE_TYPE="CI"
export IMAGE_DESCRIPTION="Airflow CI"
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh
b/scripts/ci/libraries/_push_pull_remove_images.sh
index fae0807..e853f42 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -104,7 +104,6 @@ function
push_pull_remove_images::pull_image_github_dockerhub() {
# Pulls CI image in case caching strategy is "pulled" and the image needs to
be pulled
function push_pull_remove_images::pull_ci_images_if_needed() {
-
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
@@ -122,7 +121,7 @@ Docker pulling ${PYTHON_BASE_IMAGE}.
fi
push_pull_remove_images::pull_image_github_dockerhub
"${PYTHON_BASE_IMAGE}"
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
else
- docker pull "${PYTHON_BASE_IMAGE}"
+ docker pull "${AIRFLOW_CI_PYTHON_IMAGE}"
fi
echo
fi
@@ -174,6 +173,9 @@ function
push_pull_remove_images::push_ci_images_to_dockerhub() {
# Only push default image to DockerHub registry if it is defined
push_pull_remove_images::push_image_with_retries "${DEFAULT_CI_IMAGE}"
fi
+ # Also push python image so that we use the same image as the CI image it
was built with
+ docker tag "${PYTHON_BASE_IMAGE}" "${AIRFLOW_CI_PYTHON_IMAGE}"
+ push_pull_remove_images::push_image_with_retries
"${AIRFLOW_CI_PYTHON_IMAGE}"
}
# Pushes Ci images and their tags to registry in GitHub
@@ -196,21 +198,7 @@ function
push_pull_remove_images::push_ci_images_to_github() {
PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
fi
docker tag "${PYTHON_BASE_IMAGE}"
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
- set +e
push_pull_remove_images::push_image_with_retries
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
- local result=$?
- set -e
- if [[ ${result} != "0" ]]; then
- >&2 echo
- >&2 echo "There was an unexpected error when pushing images to the
GitHub Registry"
- >&2 echo
- >&2 echo "If you see 'unknown blob' or similar kind of error it means
that it was a transient error"
- >&2 echo "And it will likely be gone next time"
- >&2 echo
- >&2 echo "Please rebase your change or 'git commit --amend; git push
--force' and try again"
- >&2 echo
- exit "${result}"
- fi
}
@@ -253,22 +241,7 @@ function
push_pull_remove_images::push_prod_images_to_github () {
# Also push prod build image
AIRFLOW_PROD_BUILD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker tag "${AIRFLOW_PROD_BUILD_IMAGE}"
"${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}"
- set +e
push_pull_remove_images::push_image_with_retries
"${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}"
- local result=$?
- set -e
- if [[ ${result} != "0" ]]; then
- >&2 echo
- >&2 echo "There was an unexpected error when pushing images to the
GitHub Registry"
- >&2 echo
- >&2 echo "If you see 'unknown blob' or similar kind of error it means
that it was a transient error"
- >&2 echo "And it will likely be gone next time"
- >&2 echo
- >&2 echo "Please rebase your change or 'git commit --amend; git push
--force' and try again"
- >&2 echo
- exit "${result}"
- fi
-
}
