This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 106d9f09852090da21f3dfb744c69e4be5fd90e5 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Wed Aug 4 23:32:12 2021 +0200 Optimizes structure of the Dockerfiles and use latest tools (#17418) * Remove CONTINUE_ON_PIP_CHECK_FAILURE parameter This parameter was useful when upgrading new dependencies, however it is going to be replaced with better approach in the upcoming image convention change. * Optimizes structure of the Dockerfiles and use latest tools This PR optimizes the structure of Dockerfile by moving some expensive operations before the COPY sources so that rebuilding image when only few sources change is much faster. At the same time, we upgrade PIP and HELM chart used to latest versions and clean-up some parameter inconsistencies. (cherry picked from commit 94b03f6f43277e7332c25fdc63aedfde605f9773) --- .github/workflows/build-images.yml | 1 - BREEZE.rst | 10 +---- Dockerfile | 27 +++++------ Dockerfile.ci | 52 +++++++++++----------- IMAGES.rst | 6 --- breeze | 9 ---- breeze-complete | 2 +- docs/docker-stack/build-arg-ref.rst | 6 --- scripts/ci/libraries/_build_images.sh | 20 --------- scripts/ci/libraries/_initialization.sh | 10 ++--- scripts/docker/install_additional_dependencies.sh | 5 +-- scripts/docker/install_airflow.sh | 4 +- ...nstall_airflow_dependencies_from_branch_tip.sh} | 11 ++--- .../docker/install_from_docker_context_files.sh | 2 +- 14 files changed, 55 insertions(+), 110 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f29e199..ec8f435 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -148,7 +148,6 @@ jobs: BACKEND: postgres PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - CONTINUE_ON_PIP_CHECK_FAILURE: "true" DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: > ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }} diff --git a/BREEZE.rst b/BREEZE.rst index 90d3f0b..468709b 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1279,9 +1279,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -2382,9 +2379,9 @@ This is the current syntax for `./breeze <./breeze>`_: Helm version - only used in case one of kind-cluster commands is used. One of: - v3.2.4 + v3.6.3 - Default: v3.2.4 + Default: v3.6.3 --executor EXECUTOR Executor to use in a kubernetes cluster. @@ -2435,9 +2432,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - **************************************************************************************************** Use different Airflow version at runtime in CI image diff --git a/Dockerfile b/Dockerfile index 66c9649..210918c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,8 @@ ARG AIRFLOW_GID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" @@ -108,12 +109,13 @@ ARG DEV_APT_COMMAND="\ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" +ARG ADDITIONAL_DEV_APT_ENV="" ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \ - ADDITIONAL_DEV_APT_ENV="" + ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV} # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 @@ -216,7 +218,7 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} @@ -236,14 +238,11 @@ ARG INSTALL_FROM_PYPI="true" # * pyjwt<2.0.0: flask-jwt-extended requires it # * dill<0.3.3 required by apache-beam ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="pyjwt<2.0.0 dill<0.3.3 certifi<2021.0.0" -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \ INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \ - EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE} + EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR /opt/airflow @@ -276,7 +275,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \ ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA} @@ -293,15 +292,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="d...@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow-build-image" \ org.opencontainers.image.title="Build Image Segment for Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow with build-time dependencies" + org.opencontainers.image.description="Reference build-time dependencies image for production-ready Apache Airflow image" ############################################################################################## # This is the actual Airflow image - much smaller than the build one. We copy @@ -379,7 +377,7 @@ ARG AIRFLOW_HOME ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED # By default PIP will install everything in ~/.local ARG PIP_USER="true" @@ -468,15 +466,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="d...@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow" \ org.opencontainers.image.title="Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow" + org.opencontainers.image.description="Reference, production-ready Apache Airflow image" ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 7b1dbf1..8d599db 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -22,6 +22,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" ARG AIRFLOW_VERSION="2.1.0.dev0" +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" + # By increasing this number we can do force build of all dependencies ARG DEPENDENCIES_EPOCH_NUMBER="6" @@ -125,6 +127,15 @@ ARG RUNTIME_APT_DEPS="\ unzip \ vim \ xxd" + +# Install Helm +ARG HELM_VERSION="v3.6.3" + +RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ + && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ + && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ + && chmod +x /usr/local/bin/helm + ARG ADDITIONAL_RUNTIME_APT_DEPS="" ARG RUNTIME_APT_COMMAND="" ARG ADDITIONAL_RUNTIME_APT_COMMAND="" @@ -207,7 +218,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" ARG INSTALL_FROM_PYPI="true" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -270,7 +281,7 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -299,8 +310,6 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. @@ -325,11 +334,13 @@ RUN chmod a+x /entrypoint COPY scripts/docker/load.bash /opt/bats/lib/ -# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not -# copying over stuff that is accidentally generated or that we do not need (such as egg-info) -# if you want to add something that is missing and you expect to see it in the image you can -# add it with ! in .dockerignore next to the airflow, test etc. directories there -COPY . ${AIRFLOW_SOURCES}/ +# Additional python deps to install +ARG ADDITIONAL_PYTHON_DEPS="" + +RUN bash /scripts/docker/install_pip_version.sh; \ + if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ + bash /scripts/docker/install_additional_dependencies.sh; \ + fi # Install autocomplete for airflow RUN if command -v airflow; then \ @@ -339,27 +350,16 @@ RUN if command -v airflow; then \ # Install autocomplete for Kubectl RUN echo "source /etc/bash_completion" >> ~/.bashrc -WORKDIR ${AIRFLOW_SOURCES} - -# Install Helm -ARG HELM_VERSION="v3.2.4" - -RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ - && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ - && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm - -# Additional python deps to install -ARG ADDITIONAL_PYTHON_DEPS="" +# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not +# copying over stuff that is accidentally generated or that we do not need (such as egg-info) +# if you want to add something that is missing and you expect to see it in the image you can +# add it with ! in .dockerignore next to the airflow, test etc. directories there +COPY . ${AIRFLOW_SOURCES}/ -RUN bash /scripts/docker/install_pip_version.sh; \ - if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - bash /scripts/docker/install_additional_dependencies.sh; \ - fi +WORKDIR ${AIRFLOW_SOURCES} ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_DATE_CREATED ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \ diff --git a/IMAGES.rst b/IMAGES.rst index 82e6989..bc34e6c 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -445,12 +445,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | upgraded to newer versions matching | | | | setup.py before installation. | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | | | | from the GitHub of Apache Airflow | | | | This allows to optimize iterations for | diff --git a/breeze b/breeze index 7decdf6..1f45ed1 100755 --- a/breeze +++ b/breeze @@ -1177,12 +1177,6 @@ function breeze::parse_arguments() { echo shift ;; - --continue-on-pip-check-failure) - export CONTINUE_ON_PIP_CHECK_FAILURE="true" - echo "Skip PIP check failure." - echo - shift - ;; --package-format) export PACKAGE_FORMAT="${2}" echo "Selected package type: ${PACKAGE_FORMAT}" @@ -2400,9 +2394,6 @@ ${FORMATTED_INSTALLATION_METHOD} --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. ---continue-on-pip-check-failure - Continue even if 'pip check' fails. - " } diff --git a/breeze-complete b/breeze-complete index db9e42a..045bb65 100644 --- a/breeze-complete +++ b/breeze-complete @@ -29,7 +29,7 @@ _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq r _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers" _breeze_allowed_kubernetes_modes="image" _breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15" -_breeze_allowed_helm_versions="v3.2.4" +_breeze_allowed_helm_versions="v3.6.3" _breeze_allowed_kind_versions="v0.11.1" _breeze_allowed_mysql_versions="5.7 8" _breeze_allowed_postgres_versions="9.6 10 11 12 13" diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 8780970..f2507e0 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -79,12 +79,6 @@ for examples of using those arguments. +------------------------------------------+------------------------------------------+------------------------------------------+ | Build argument | Default value | Description | +==========================================+==========================================+==========================================+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are | | | | upgraded to newer versions matching | | | | setup.py before installation. | diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index ca94e4c..4ead442 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -669,7 +669,6 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="${DEFAULT_CONSTRAINTS_BRANCH}" \ --build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \ @@ -810,7 +809,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ @@ -845,7 +843,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -1014,23 +1011,6 @@ ${COLOR_BLUE} ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 ${COLOR_RESET} -* If you want to build the image regardless if 'pip check' fails for it, you can add - --continue-on-pip-check-failure flag and enter the image and inspect dependencies. - -CI image: - -${COLOR_BLUE} - ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6-ci bash -${COLOR_RESET} - -Production image: - -${COLOR_BLUE} - ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6 bash -${COLOR_RESET} - * You will see error messages there telling which requirements are conflicting and which packages caused the conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS variable in Dockerfile.ci. Note that the limitations might be different for Dockerfile.ci and Dockerfile diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index cbeb932..2cebe7d 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -200,7 +200,7 @@ function initialization::initialize_files_for_rebuild_check() { "scripts/docker/common.sh" "scripts/docker/install_additional_dependencies.sh" "scripts/docker/install_airflow.sh" - "scripts/docker/install_airflow_from_branch_tip.sh" + "scripts/docker/install_airflow_dependencies_from_branch_tip.sh" "scripts/docker/install_from_docker_context_files.sh" "scripts/docker/install_mysql.sh" "airflow/www/package.json" @@ -401,7 +401,7 @@ function initialization::initialize_image_build_variables() { export INSTALLED_PROVIDERS export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.1"} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.2"} export AIRFLOW_PIP_VERSION # We also pin version of wheel used to get consistent builds @@ -425,9 +425,6 @@ function initialization::initialize_image_build_variables() { # Installs different airflow version than current from the sources export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - # Continue on PIP CHECK failure - export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"} - # Determines if airflow should be installed from a specified reference in GitHub export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} @@ -482,7 +479,7 @@ function initialization::initialize_kubernetes_variables() { CURRENT_KIND_VERSIONS+=("v0.11.1") export CURRENT_KIND_VERSIONS # Currently supported versions of Helm - CURRENT_HELM_VERSIONS+=("v3.2.4") + CURRENT_HELM_VERSIONS+=("v3.6.3") export CURRENT_HELM_VERSIONS # Current executor in chart CURRENT_EXECUTOR+=("KubernetesExecutor") @@ -663,7 +660,6 @@ Common image build variables: INSTALL_FROM_PYPI: '${INSTALL_FROM_PYPI}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' - CONTINUE_ON_PIP_CHECK_FAILURE: '${CONTINUE_ON_PIP_CHECK_FAILURE}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' AIRFLOW_CONSTRAINTS_LOCATION: '${AIRFLOW_CONSTRAINTS_LOCATION}' AIRFLOW_CONSTRAINTS_REFERENCE: '${AIRFLOW_CONSTRAINTS_REFERENCE}' diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 6c035ae..4f9c05f 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -23,7 +23,6 @@ test -v ADDITIONAL_PYTHON_DEPS test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS test -v AIRFLOW_INSTALL_USER_FLAG test -v AIRFLOW_PIP_VERSION -test -v CONTINUE_ON_PIP_CHECK_FAILURE # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -41,7 +40,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else echo echo Installing additional dependencies upgrading only if needed @@ -51,7 +50,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 4904027..e2bca4f 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -60,7 +60,7 @@ function install_airflow() { # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else \ echo echo Installing all packages with constraints and upgrade if needed @@ -76,7 +76,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh similarity index 85% rename from scripts/docker/install_airflow_from_branch_tip.sh rename to scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 925a872..61aaa13 100755 --- a/scripts/docker/install_airflow_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -30,28 +30,29 @@ . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -function install_airflow_from_branch_tip() { +function install_airflow_dependencies_from_branch_tip() { echo echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies" echo if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} fi - # Install latest set of dependencies using constraints + # Install latest set of dependencies using constraints. In case constraints were upgraded and there + # are conflicts, this might fail, but it should be fixed in the following installation steps pip install ${AIRFLOW_INSTALL_USER_FLAG} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true echo echo Uninstalling just airflow. Dependencies remain. echo - pip uninstall --yes apache-airflow + pip uninstall --yes apache-airflow || true } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -install_airflow_from_branch_tip +install_airflow_dependencies_from_branch_tip diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 813d1b0..d8ed6bc 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -96,7 +96,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # make sure correct PIP version is left installed pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check }