This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push: new 61b4482 Fixes force-pulling base python images (#14736) 61b4482 is described below commit 61b448221d65aae80c89b3a84376b6201319ff33 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Mar 13 19:38:49 2021 +0100 Fixes force-pulling base python images (#14736) Sometimes base python image patchlevel might case failure of tests. This happens for example with test_views.py tests fixed in #14719 where CVE fix in all python versions caused our tests to fail. There was an error in our scripts - when --force-pull-images were used, the base python version was not updated to the latest version even if there was a newer one and it caused our images to bounce few times between two latest patchlevels when they were manually refreshed. This change fixes it so that the base python image is always used when a) FORCE_PULL_IMAGES is true or b) UPGRADE_TO_NEWER_DEPENDENCIES is != false This will cause python upgrade in two cases: - when images are rebuilt with --force-pull-images locally - when images are upgraded with newer dependencies on master --- .github/workflows/build-images-workflow-run.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/scheduled_quarantined.yml | 2 +- scripts/ci/libraries/_push_pull_remove_images.sh | 27 ++++++++++++------------ 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index 33c7a79..65ba7aa 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -24,7 +24,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5e115b..c0523d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml index 00663cb..2f7f05f 100644 --- a/.github/workflows/scheduled_quarantined.yml +++ b/.github/workflows/scheduled_quarantined.yml @@ -25,7 +25,7 @@ on: # yamllint disable-line rule:truthy env: MOUNT_SELECTED_LOCAL_SOURCES: "false" FORCE_ANSWER_TO_QUESTIONS: "yes" - FORCE_PULL_IMAGES: "true" + FORCE_PULL_IMAGES: "false" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index c48491d..5090c5b 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -104,33 +104,34 @@ function push_pull_remove_images::pull_image_github_dockerhub() { set -e } +# Force pulls the python base image +function push_pull_remove_images::force_pull_python_base_image() { + docker pull "${PYTHON_BASE_IMAGE}" + echo "FROM ${PYTHON_BASE_IMAGE}" | \ + docker build \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - +} + # Pulls the base Python image. This image is used as base for CI and PROD images, depending on the parameters used: # -# * if UPGRADE_TO_NEWER_DEPENDENCIES is noy false, then it pulls the latest Python image available first and +# * if FORCE_PULL_IMAGES is true or UPGRADE_TO_NEWER_DEPENDENCIES != false, then it pulls the latest Python image available first and # adds `org.opencontainers.image.source` label to it, so that it is linked to Airflow repository when # we push it to GHCR registry -# * If UPGRADE_TO_NEWER_DEPENDENCIES it pulls the Python base image from either GitHub registry or from DockerHub +# * Otherwise it pulls the Python base image from either GitHub registry or from DockerHub # depending on USE_GITHUB_REGISTRY variable. In case we pull specific build image (via suffix) # it will pull the right image using the specified suffix function push_pull_remove_images::pull_base_python_image() { echo - echo "Force pull base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}" + echo "Force pull python base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}" echo if [[ -n ${DETECTED_TERMINAL=} ]]; then echo -n " Docker pulling ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies ${UPGRADE_TO_NEWER_DEPENDENCIES} " > "${DETECTED_TERMINAL}" fi - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then - # Pull latest PYTHON_BASE_IMAGE, so that it is linked to the current project it is build in. - # This is necessary in case we use Google Container registry - we always use the - # Airflow version of the python image with the opencontainers label, so that GHCR can link it - # to the Apache Airflow repository. - docker pull "${PYTHON_BASE_IMAGE}" - echo "FROM ${PYTHON_BASE_IMAGE}" | \ - docker build \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - -t "${AIRFLOW_PYTHON_BASE_IMAGE}" - + if [[ "${FORCE_PULL_IMAGES}" == "true" || ${UPGRADE_TO_NEWER_DEPENDENCIES} != "false" ]]; then + push_pull_remove_images::force_pull_python_base_image else if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then PYTHON_TAG_SUFFIX=""