This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4b642c10d34d34989732d63846ce52b63fdf9006 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Sep 27 18:00:03 2020 +0200 Optionally disables PIP cache from GitHub during the build (#11173) This is first step of implementing the corporate-environment friendly way of building images, where in the corporate environment, this might not be possible to install the packages using the GitHub cache initially. Part of #11171 (cherry picked from commit f16354bc02f563bcaee704cafe666c594fbec5e8) --- BREEZE.rst | 9 +++++++++ Dockerfile | 11 ++++++++--- Dockerfile.ci | 11 ++++++++--- IMAGES.rst | 8 ++++++++ breeze | 10 ++++++++++ breeze-complete | 2 +- docs/production-deployment.rst | 8 ++++++++ scripts/ci/libraries/_build_images.sh | 3 +++ scripts/ci/libraries/_initialization.sh | 4 +++- 9 files changed, 58 insertions(+), 8 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 728d590..11f7cdb 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1218,6 +1218,9 @@ This is the current syntax for `./breeze <./breeze>`_: --additional-runtime-deps Additional apt runtime dependencies to use when building the images. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1639,6 +1642,9 @@ This is the current syntax for `./breeze <./breeze>`_: --additional-runtime-deps Additional apt runtime dependencies to use when building the images. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1946,6 +1952,9 @@ This is the current syntax for `./breeze <./breeze>`_: --additional-runtime-deps Additional apt runtime dependencies to use when building the images. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/Dockerfile b/Dockerfile index a5db8e5..8ad448f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -163,11 +163,16 @@ ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL} ENV PATH=${PATH}:/root/.local/bin RUN mkdir -p /root/.local/bin +ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" +ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} + # In case of Production build image segment we want to pre-install master version of airflow # dependencies from github so that we do not have to always reinstall it from the scratch. -RUN pip install --user \ - "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ - --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; +RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ + pip install --user \ + "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ + --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \ + fi ARG AIRFLOW_SOURCES_FROM="." ENV AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM} diff --git a/Dockerfile.ci b/Dockerfile.ci index d6beac6..b100f17 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -216,13 +216,18 @@ ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL} ARG AIRFLOW_CI_BUILD_EPOCH="1" ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} +ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" +ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} + # In case of CI builds we want to pre-install master version of airflow dependencies so that # We do not have to always reinstall it from the scratch. # This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER. # And is automatically reinstalled from the scratch every time patch release of python gets released -RUN pip install \ - "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ - --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; +RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ + pip install \ + "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]"; \ + --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \ + fi # Link dumb-init for backwards compatibility (so that older images also work) diff --git a/IMAGES.rst b/IMAGES.rst index bd7dcba..6599d96 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -309,6 +309,14 @@ The following build arguments (``--build-arg`` in docker build command) can be u +------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_EXTRAS`` | ``all`` | extras to install | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | +| | | from the GitHub of Apache Airflow | +| | | This allows to optimize iterations for | +| | | Image builds and speeds up CI builds | +| | | But in some corporate environments it | +| | | might be forbidden to download anything | +| | | from public repositories. | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``ADDITIONAL_AIRFLOW_EXTRAS`` | | additional extras to install | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``ADDITIONAL_PYTHON_DEPS`` | | additional python dependencies to | diff --git a/breeze b/breeze index 56173aa..6c476a7 100755 --- a/breeze +++ b/breeze @@ -905,6 +905,13 @@ function breeze::parse_arguments() { # if not set here, docker cached is determined later, depending on type of image to be build shift ;; + -B | --disable-pip-cache) + echo "Disable PIP cache during build" + echo + export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" + shift + ;; + -P | --force-pull-images) echo "Force pulling images before build. Uses pulled images as cache." echo @@ -2097,6 +2104,9 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} --additional-runtime-deps Additional apt runtime dependencies to use when building the images. +--disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/breeze-complete b/breeze-complete index e8b281a..9abb033 100644 --- a/breeze-complete +++ b/breeze-complete @@ -131,7 +131,7 @@ kubernetes-mode: kubernetes-version: helm-version: kind-version: skip-mounting-local-sources install-airflow-version: install-airflow-reference: db-reset verbose assume-yes assume-no assume-quit forward-credentials start-airflow rbac-ui init-script: force-build-images force-pull-images production-image extras: force-clean-images skip-rebuild-check -build-cache-local build-cache-pulled build-cache-disabled +build-cache-local build-cache-pulled build-cache-disabled disable-pip-cache dockerhub-user: dockerhub-repo: github-registry github-repository: github-image-id: postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: diff --git a/docs/production-deployment.rst b/docs/production-deployment.rst index 9f33e5c..aedb752 100644 --- a/docs/production-deployment.rst +++ b/docs/production-deployment.rst @@ -289,6 +289,14 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``AIRFLOW_EXTRAS`` | (see Dockerfile) | Default extras with which airflow is | | | | installed | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | +| | | from the GitHub of Apache Airflow | +| | | This allows to optimize iterations for | +| | | Image builds and speeds up CI builds | +| | | But in some corporate environments it | +| | | might be forbidden to download anything | +| | | from public repositories. | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``ADDITIONAL_AIRFLOW_EXTRAS`` | | Optional additional extras with which | | | | airflow is installed | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index cbc8964..41856d8 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -556,6 +556,7 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ @@ -685,6 +686,7 @@ function build_images::build_prod_images() { --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ @@ -705,6 +707,7 @@ function build_images::build_prod_images() { --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \ diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 419a6fe..afc8bee 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -320,7 +320,8 @@ function initialization::initialize_image_build_variables() { export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}" # additional runtime apt dependencies on top of the default ones export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}" - + # whether pre cached pip packages are used during build + export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}" } # Determine version suffixes used to build backport packages @@ -637,6 +638,7 @@ function initialization::make_constants_read_only() { readonly ADDITIONAL_PYTHON_DEPS readonly ADDITIONAL_DEV_DEPS readonly ADDITIONAL_RUNTIME_DEPS + readonly AIRFLOW_PRE_CACHED_PIP_PACKAGES readonly DOCKERHUB_USER readonly DOCKERHUB_REPO
