This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit fa6c547a0ad42c47f4826a3e88b3d1576d41c0b1 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Dec 6 23:36:33 2020 +0100 Production images on CI are now built from packages (#12685) So far, the production images of Airflow were using sources when they were built on CI. This PR changes that, to build airflow + providers packages first and install them rather than use sources as installation mechanism. Part of #12261 (cherry picked from commit ed1825c0264d1f77c4754b722fb3721cbcd779d7) --- .dockerignore | 3 +- .github/workflows/build-images-workflow-run.yml | 6 +-- .github/workflows/scheduled_quarantined.yml | 1 - BREEZE.rst | 8 ++++ CI.rst | 19 ++------- CONTRIBUTING.rst | 2 +- Dockerfile.ci | 16 ++++---- LOCAL_VIRTUALENV.rst | 25 ++++++++++++ breeze | 2 + breeze-complete | 4 +- docker-context-files/README.md | 2 +- .../ci_build_airflow_package.sh} | 18 ++++++--- scripts/ci/docker-compose/files.yml | 1 + scripts/ci/docker-compose/local.yml | 1 - scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 2 + scripts/ci/images/ci_verify_prod_image.sh | 2 +- scripts/ci/libraries/_all_libs.sh | 2 + .../libraries/_build_airflow_packages.sh} | 32 ++++++++++----- scripts/ci/libraries/_initialization.sh | 22 ++++------- scripts/ci/libraries/_local_mounts.sh | 1 - scripts/ci/libraries/_runs.sh | 17 ++++++-- scripts/ci/testing/ci_run_airflow_testing.sh | 5 +-- scripts/in_container/_in_container_utils.sh | 16 -------- scripts/in_container/check_environment.sh | 46 ++++++++++++---------- scripts/in_container/configure_environment.sh | 1 - scripts/in_container/entrypoint_exec.sh | 1 - tests/bats/breeze/test_breeze_complete.bats | 8 ++++ 27 files changed, 152 insertions(+), 111 deletions(-) diff --git a/.dockerignore b/.dockerignore index ac7372b..8a90d74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -59,8 +59,9 @@ !.github !empty -# This folder is for you if you want to add any files to the docker context when you build your own +# This folder is for you if you want to add any packages to the docker context when you build your own # docker image. most of other files and any new folder you add will be excluded by default +# if you need other types of files - please add the extensions here. !docker-context-files # Avoid triggering context change on README change (new companies using Airflow) diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index c5480c6..93b2176 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -23,7 +23,6 @@ on: # yamllint disable-line rule:truthy types: ['requested'] env: MOUNT_LOCAL_SOURCES: "false" - MOUNT_FILES: "true" FORCE_ANSWER_TO_QUESTIONS: "yes" FORCE_PULL_IMAGES: "true" CHECK_IMAGE_FOR_REBUILD: "true" @@ -328,7 +327,7 @@ jobs: - name: "Setup python" uses: actions/setup-python@v2 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ matrix.python-version }} if: steps.defaults.outputs.proceed == 'true' - name: > Override "scripts/ci" with the "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch @@ -345,7 +344,8 @@ jobs: if: steps.defaults.outputs.proceed == 'true' - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' + # locally built CI image is needed to prepare packages for PROD image build + if: steps.defaults.outputs.proceed == 'true' - name: "Push CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_push_ci_images.sh if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml index 14dc91d..9877e4c 100644 --- a/.github/workflows/scheduled_quarantined.yml +++ b/.github/workflows/scheduled_quarantined.yml @@ -24,7 +24,6 @@ on: # yamllint disable-line rule:truthy env: MOUNT_LOCAL_SOURCES: "false" - MOUNT_FILES: "true" FORCE_ANSWER_TO_QUESTIONS: "yes" FORCE_PULL_IMAGES: "true" CHECK_IMAGE_FOR_REBUILD: "true" diff --git a/BREEZE.rst b/BREEZE.rst index 43018a0..4c1b3a7 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1653,6 +1653,14 @@ This is the current syntax for `./breeze <./breeze>`_: Default: + -S, --version-suffix-for-pypi SUFFIX + Adds optional suffix to the version in the generated backport package. It can be used + to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI. + + -N, --version-suffix-for-svn SUFFIX + Adds optional suffix to the generated names of package. It can be used to generate + rc1/rc2 ... versions of the packages to be uploaded to SVN. + -v, --verbose Show verbose information about executed docker, kind, kubectl, helm commands. Useful for debugging - when you run breeze with --verbose flags you will be able to see the commands diff --git a/CI.rst b/CI.rst index e087438..0ac1c9d 100644 --- a/CI.rst +++ b/CI.rst @@ -135,13 +135,6 @@ You can use those variables when you try to reproduce the build locally. | | | | | directories) generated locally on the | | | | | | host during development. | +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ -| ``MOUNT_FILES`` | true | true | true | Determines whether "files" folder from | -| | | | | sources is mounted as "/files" folder | -| | | | | inside the container. This is used to | -| | | | | share results of local actions to the | -| | | | | host, as well as to pass host files to | -| | | | | inside container for local development. | -+-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ | Force variables | +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ | ``FORCE_PULL_IMAGES`` | true | true | true | Determines if images are force-pulled, | @@ -203,7 +196,9 @@ You can use those variables when you try to reproduce the build locally. | Image variables | +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ | ``INSTALL_AIRFLOW_VERSION`` | | | | Installs Airflow version from PyPI when | -| | | | | building image. | +| | | | | building image. Can be "none" to skip airflow | +| | | | | installation so that it can be installed from | +| | | | | locally prepared packages. | +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ | ``INSTALL_AIRFLOW_REFERENCE`` | | | | Installs Airflow version from GitHub | | | | | | branch or tag. | @@ -362,12 +357,6 @@ Note that you need to set "CI" variable to true in order to get the same results | | | [``pull_request``, ``pull_request_target``, | | | | ``schedule``, ``push``] | +------------------------------+----------------------+-----------------------------------------------------+ -| CI_SOURCE_REPO | ``apache/airflow`` | Source repository. This might be different than the | -| | | ``CI_TARGET_REPO`` for pull requests | -+------------------------------+----------------------+-----------------------------------------------------+ -| CI_SOURCE_BRANCH | ``master`` | Branch in the source repository that is used to | -| | | make the pull request. | -+------------------------------+----------------------+-----------------------------------------------------+ | CI_REF | ``refs/head/master`` | Branch in the source repository that is used to | | | | make the pull request. | +------------------------------+----------------------+-----------------------------------------------------+ @@ -700,7 +689,7 @@ We also have a script that can help to clean-up the old artifacts: CodeQL scan ----------- -The CodeQL security scan uses GitHub security scan framework to scan our code for security violations. +The `CodeQL <https://securitylab.github.com/tools/codeql>`_ security scan uses GitHub security scan framework to scan our code for security violations. It is run for JavaScript and python code. Naming conventions for stored images diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index bc5661c..bc64117 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -579,7 +579,7 @@ and not installed together with the core, unless you set ``INSTALL_PROVIDERS_FRO variable to ``true``. In Breeze - which is a development environment, ``INSTALL_PROVIDERS_FROM_SOURCES`` variable is set to true, -but you can add ``--skip-installing-airflow-providers`` flag to Breeze to skip installing providers when +but you can add ``--skip-installing-airflow-providers-from-sources`` flag to Breeze to skip installing providers when building the images. One watch-out - providers are still always installed (or rather available) if you install airflow from diff --git a/Dockerfile.ci b/Dockerfile.ci index 2210989..67e5bb1 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -261,11 +261,11 @@ ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} -ARG AIRFLOW_LOCAL_PIP_WHEELS="" -ENV AIRFLOW_LOCAL_PIP_WHEELS=${AIRFLOW_LOCAL_PIP_WHEELS} +ARG INSTALL_FROM_DOCKER_CONTEXT_FILES="" +ENV INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} -ARG INSTALL_AIRFLOW_VIA_PIP="true" -ENV INSTALL_AIRFLOW_VIA_PIP=${INSTALL_AIRFLOW_VIA_PIP} +ARG INSTALL_FROM_PYPI="true" +ENV INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} RUN pip install --upgrade "pip==${PIP_VERSION}" @@ -317,7 +317,7 @@ ENV UPGRADE_TO_LATEST_CONSTRAINTS=${UPGRADE_TO_LATEST_CONSTRAINTS} # Usually we will install versions based on the dependencies in setup.py and upgraded only if needed. # But in cron job we will install latest versions matching setup.py to see if there is no breaking change # and push the constraints if everything is successful -RUN if [[ ${INSTALL_AIRFLOW_VIA_PIP} == "true" ]]; then \ +RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS}" != "false" ]]; then \ pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \ else \ @@ -329,9 +329,9 @@ RUN if [[ ${INSTALL_AIRFLOW_VIA_PIP} == "true" ]]; then \ # they are also installed additionally to whatever is installed from Airflow. COPY docker-context-files/ /docker-context-files/ -RUN if [[ ${AIRFLOW_LOCAL_PIP_WHEELS} != "true" ]]; then \ - if ls /docker-context-files/*.whl 1> /dev/null 2>&1; then \ - pip install --no-deps /docker-context-files/*.whl; \ +RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} != "true" ]]; then \ + if ls /docker-context-files/*.{whl,tar.gz} 1> /dev/null 2>&1; then \ + pip install --no-deps /docker-context-files/*.{whl,tar.gz}; \ fi ; \ fi diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 574366d..03b60c8 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -171,6 +171,31 @@ Activate your virtualenv, e.g. by using ``workon``, and once you are in it, run: cd airflow/www yarn build +Developing Providers +-------------------- + +In Airflow 2.0 we introduced split of Apache Airflow into separate packages - there is one main +apache-airflow package with core of Airflow and 70+ packages for all providers (external services +and software Airflow can communicate with). + +Developing providers is part of Airflow development, but when you install airflow as editable in your local +development environment, the corresponding provider packages will be also installed from PyPI. However, the +providers will also be present in your "airflow/providers" folder. This might lead to confusion, +which sources of providers are imported during development. It will depend on your +environment's PYTHONPATH setting in general. + +In order to avoid the confusion, you can set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment to ``true`` +before running ``pip install`` command: + +.. code-block:: bash + + INSTALL_PROVIDERS_FROM_SOURCES="true" pip install -U -e ".[devel,<OTHER EXTRAS>]" \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-master/constraints-3.6.txt"; + +This way no providers packages will be installed and they will always be imported from the "airflow/providers" +folder. + + Running Tests ------------- diff --git a/breeze b/breeze index f6a45a5..7a26b30 100755 --- a/breeze +++ b/breeze @@ -1676,6 +1676,7 @@ ${CMDNAME} prepare-airflow-packages [FLAGS] Flags: $(breeze::flag_packages) +$(breeze::flag_version_suffix) $(breeze::flag_verbosity) " readonly DETAILED_USAGE_PREPARE_AIRFLOW_PACKAGES @@ -2140,6 +2141,7 @@ function breeze::flag_local_file_mounting() { -l, --skip-mounting-local-sources Skips mounting local volume with sources - you get exactly what is in the docker image rather than your current local sources of Airflow. + " } diff --git a/breeze-complete b/breeze-complete index 7e1ccc6..042a7fa 100644 --- a/breeze-complete +++ b/breeze-complete @@ -140,12 +140,12 @@ build-cache-local build-cache-pulled build-cache-disabled disable-pip-cache dockerhub-user: dockerhub-repo: github-registry github-repository: github-image-id: postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: image-tag: -disable-mysql-client-installation constraints-location: disable-pip-cache add-local-pip-wheels +disable-mysql-client-installation constraints-location: disable-pip-cache install-from-docker-context-files additional-extras: additional-python-deps: skip-installing-airflow-via-pip dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env: runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags -install-wheels no-rbac-ui +install-packages-from-dist no-rbac-ui package-format: test-type: preserve-volumes " diff --git a/docker-context-files/README.md b/docker-context-files/README.md index 52213cb..07a6c22 100644 --- a/docker-context-files/README.md +++ b/docker-context-files/README.md @@ -26,7 +26,7 @@ th [docker-context-files](.) folder to the image context - in case of production the build segment, co content of the folder is available in the `/docker-context-file` folder inside the build image. You can store constraint files and wheel packages there that you want to install as PYPI packages and refer to those packages using -`--constraint-location` flag for constraints or by using `--add-local-pip-wheels` flag. +`--constraint-location` flag for constraints or by using `--install-from-local-files-when-building` flag. By default, the content of this folder is .gitignored so that any binaries and files you put here are only used for local builds and not committed to the repository. diff --git a/scripts/ci/docker-compose/files.yml b/scripts/ci/build_airflow/ci_build_airflow_package.sh old mode 100644 new mode 100755 similarity index 67% copy from scripts/ci/docker-compose/files.yml copy to scripts/ci/build_airflow/ci_build_airflow_package.sh index 5625ca6..593d94a --- a/scripts/ci/docker-compose/files.yml +++ b/scripts/ci/build_airflow/ci_build_airflow_package.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,9 +15,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ---- -version: "2.2" -services: - airflow: - volumes: - - ../../../files:/files:cached +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +build_airflow_packages::build_airflow_packages + +cd "${AIRFLOW_SOURCES}/dist" || exit 1 + +dump_file="/tmp/airflow_$(date +"%Y%m%d-%H%M%S").tar.gz" +tar -cvzf "${dump_file}" . + +echo "Airflow is in dist and also tar-gzipped in ${dump_file}" diff --git a/scripts/ci/docker-compose/files.yml b/scripts/ci/docker-compose/files.yml index 5625ca6..2925bd9 100644 --- a/scripts/ci/docker-compose/files.yml +++ b/scripts/ci/docker-compose/files.yml @@ -20,3 +20,4 @@ services: airflow: volumes: - ../../../files:/files:cached + - ../../../dist:/dist:cached diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 20ce4d3..1fb6200 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -43,7 +43,6 @@ services: - ../../../dags:/opt/airflow/dags:cached - ../../../dev:/opt/airflow/dev:cached - ../../../docs:/opt/airflow/docs:cached - - ../../../dist:/dist:cached - ../../../hooks:/opt/airflow/hooks:cached - ../../../logs:/root/airflow/logs:cached - ../../../pytest.ini:/opt/airflow/pytest.ini:cached diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh index 700487c..43defdf 100755 --- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh @@ -23,6 +23,8 @@ export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" export DOCKER_CACHE="local" export VERBOSE="true" +export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" +readonly INSTALLED_EXTRAS # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" diff --git a/scripts/ci/images/ci_verify_prod_image.sh b/scripts/ci/images/ci_verify_prod_image.sh index 30f9def..ec529b4 100755 --- a/scripts/ci/images/ci_verify_prod_image.sh +++ b/scripts/ci/images/ci_verify_prod_image.sh @@ -62,7 +62,7 @@ function verify_prod_image_dependencies { docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' local res=$? if [[ ${res} != "0" ]]; then - echo "${COLOR_RED_ERROR} ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}" + echo "${COLOR_RED_ERROR} ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}" echo build_images::inform_about_pip_check "--production " # TODO(potiuk) - enable the comment once https://github.com/apache/airflow/pull/12188 is merged diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh index 43e2e23..d676e77 100755 --- a/scripts/ci/libraries/_all_libs.sh +++ b/scripts/ci/libraries/_all_libs.sh @@ -32,6 +32,8 @@ readonly SCRIPTS_CI_DIR . "${LIBRARIES_DIR}"/_repeats.sh # shellcheck source=scripts/ci/libraries/_sanity_checks.sh . "${LIBRARIES_DIR}"/_sanity_checks.sh +# shellcheck source=scripts/ci/libraries/_build_airflow_packages.sh +. "${LIBRARIES_DIR}"/_build_airflow_packages.sh # shellcheck source=scripts/ci/libraries/_build_images.sh . "${LIBRARIES_DIR}"/_build_images.sh # shellcheck source=scripts/ci/libraries/_kind.sh diff --git a/scripts/in_container/entrypoint_exec.sh b/scripts/ci/libraries/_build_airflow_packages.sh old mode 100755 new mode 100644 similarity index 54% copy from scripts/in_container/entrypoint_exec.sh copy to scripts/ci/libraries/_build_airflow_packages.sh index 728872b..3ec1bca --- a/scripts/in_container/entrypoint_exec.sh +++ b/scripts/ci/libraries/_build_airflow_packages.sh @@ -16,16 +16,30 @@ # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/in_container/_in_container_script_init.sh -. /opt/airflow/scripts/in_container/_in_container_script_init.sh +# Build airflow packages +function build_airflow_packages::build_airflow_packages() { + rm -rf -- *egg-info* + rm -rf -- build -# shellcheck source=scripts/in_container/configure_environment.sh -. "$( dirname "${BASH_SOURCE[0]}" )/configure_environment.sh" + pip install --upgrade "pip==${PIP_VERSION}" "wheel==${WHEEL_VERSION}" -# shellcheck source=scripts/in_container/run_init_script.sh -. "$( dirname "${BASH_SOURCE[0]}" )/run_init_script.sh" + local packages=() -# shellcheck source=scripts/in_container/run_tmux.sh -. "$( dirname "${BASH_SOURCE[0]}" )/run_tmux.sh" + if [[ ${PACKAGE_FORMAT} == "wheel" || ${PACKAGE_FORMAT} == "both" ]] ; then + packages+=("bdist_wheel") + fi + if [[ ${PACKAGE_FORMAT} == "sdist" || ${PACKAGE_FORMAT} == "both" ]] ; then + packages+=("sdist") + fi -exec /bin/bash "${@}" + # Prepare airflow's wheel + python setup.py compile_assets "${packages[@]}" + + # clean-up + rm -rf -- *egg-info* + rm -rf -- build + + echo + echo "Airflow package prepared: ${PACKAGE_FORMAT}" + echo +} diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 50cdbf0..40437dd 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -126,9 +126,9 @@ function initialization::initialize_base_variables() { # If set to true, RBAC UI will not be used for 1.10 version export DISABLE_RBAC=${DISABLE_RBAC:="false"} - # if set to true, the ci image will look for wheel packages in dist folder and will install them + # if set to true, the ci image will look for packages in dist folder and will install them # during entering the container - export INSTALL_WHEELS=${INSTALL_WHEELS:="false"} + export INSTALL_PACKAGES_FROM_DIST=${INSTALL_PACKAGES_FROM_DIST:="false"} # If set the specified file will be used to initialize Airflow after the environment is created, # otherwise it will use files/airflow-breeze-config/init.sh @@ -235,9 +235,6 @@ function initialization::initialize_mount_variables() { # Whether necessary for airflow run local sources are mounted to docker export MOUNT_LOCAL_SOURCES=${MOUNT_LOCAL_SOURCES:="true"} - # Whether files folder from local sources are mounted to docker - export MOUNT_FILES=${MOUNT_FILES:="true"} - if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then verbosity::print_info verbosity::print_info "Mounting necessary host volumes to Docker" @@ -249,14 +246,9 @@ function initialization::initialize_mount_variables() { verbosity::print_info fi - if [[ ${MOUNT_FILES} == "true" ]]; then - verbosity::print_info - verbosity::print_info "Mounting files folder to Docker" - verbosity::print_info - EXTRA_DOCKER_FLAGS+=("-v" "${AIRFLOW_SOURCES}/files:/files") - fi - EXTRA_DOCKER_FLAGS+=( + "-v" "${AIRFLOW_SOURCES}/files:/files" + "-v" "${AIRFLOW_SOURCES}/dist:/dist" "--rm" "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" ) @@ -383,10 +375,10 @@ function initialization::initialize_image_build_variables() { # whether installation of Airflow should be done via PIP. You can set it to false if you have # all the binary packages (including airflow) in the docker-context-files folder and use - # AIRFLOW_LOCAL_PIP_WHEELS="true" to install it from there. - export INSTALL_AIRFLOW_VIA_PIP="${INSTALL_AIRFLOW_VIA_PIP:="true"}" + # INSTALL_FROM_DOCKER_CONTEXT_FILES="true" to install it from there. + export INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI:="true"}" # whether installation should be performed from the local wheel packages in "docker-context-files" folder - export AIRFLOW_LOCAL_PIP_WHEELS="${AIRFLOW_LOCAL_PIP_WHEELS:="false"}" + export INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES:="false"}" # reference to CONSTRAINTS. they can be overwritten manually or replaced with AIRFLOW_CONSTRAINTS_LOCATION export AIRFLOW_CONSTRAINTS_REFERENCE="${AIRFLOW_CONSTRAINTS_REFERENCE:=""}" # direct constraints Location - can be URL or path to local file. If empty, it will be calculated diff --git a/scripts/ci/libraries/_local_mounts.sh b/scripts/ci/libraries/_local_mounts.sh index 39790c9..0ccd250 100644 --- a/scripts/ci/libraries/_local_mounts.sh +++ b/scripts/ci/libraries/_local_mounts.sh @@ -39,7 +39,6 @@ function local_mounts::generate_local_mounts_list { "$prefix"dags:/opt/airflow/dags:cached "$prefix"dev:/opt/airflow/dev:cached "$prefix"docs:/opt/airflow/docs:cached - "$prefix"dist:/dist:cached "$prefix"hooks:/opt/airflow/hooks:cached "$prefix"logs:/root/airflow/logs:cached "$prefix"pytest.ini:/opt/airflow/pytest.ini:cached diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index 7b4e51e..17ab1ee 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -19,9 +19,10 @@ # Docker command to build documentation function runs::run_docs() { docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint "/usr/local/bin/dumb-init" \ - "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/in_container/run_docs_build.sh" "${@}" + -e "GITHUB_ACTIONS=${GITHUB_ACTIONS="false"}" \ + --entrypoint "/usr/local/bin/dumb-init" \ + "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/in_container/run_docs_build.sh" "${@}" } @@ -32,3 +33,13 @@ function runs::run_generate_constraints() { "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/in_container/run_generate_constraints.sh" } + +# Docker command to prepare provider packages +function runs::run_prepare_airflow_packages() { + docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/usr/local/bin/dumb-init" \ + -t \ + -v "${AIRFLOW_SOURCES}:/opt/airflow" \ + "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh" "${@}" +} diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 39f1501..8bc6169 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -103,15 +103,12 @@ build_images::rebuild_ci_image_if_needed initialization::set_mysql_encoding -DOCKER_COMPOSE_LOCAL=() +DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml") if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") fi -if [[ ${MOUNT_FILES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml") -fi if [[ ${GITHUB_ACTIONS} == "true" ]]; then DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/ga.yml") diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 1dbcdd4..f1a55c8 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -260,22 +260,6 @@ function uninstall_airflow() { find /root/airflow/ -type f -print0 | xargs -0 rm -f -- } -function uninstall_providers() { - echo - echo "Uninstalling all provider packages" - echo - local provider_packages_to_uninstall - provider_packages_to_uninstall=$(pip freeze | grep apache-airflow-providers || true) - if [[ -n ${provider_packages_to_uninstall} ]]; then - echo "${provider_packages_to_uninstall}" | xargs pip uninstall -y || true 2>/dev/null - fi -} - -function uninstall_airflow_and_providers() { - uninstall_providers - uninstall_airflow -} - function install_released_airflow_version() { local version="${1}" local extras="${2}" diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index 9ab400a..258036f 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -20,13 +20,14 @@ EXIT_CODE=0 DISABLED_INTEGRATIONS="" -function check_service() { - INTEGRATION_NAME=$1 +function check_service { + LABEL=$1 CALL=$2 MAX_CHECK=${3:=1} - echo -n "${INTEGRATION_NAME}: " - while true; do + echo -n "${LABEL}: " + while true + do set +e LAST_CHECK_RESULT=$(eval "${CALL}" 2>&1) RES=$? @@ -36,7 +37,7 @@ function check_service() { break else echo -n "." - MAX_CHECK=$((MAX_CHECK - 1)) + MAX_CHECK=$((MAX_CHECK-1)) fi if [[ ${MAX_CHECK} == 0 ]]; then echo "${COLOR_RED_ERROR} Maximum number of retries while checking service. Exiting ${COLOR_RESET}" @@ -55,8 +56,11 @@ function check_service() { fi } -function check_integration() { - INTEGRATION_NAME=$1 +function check_integration { + INTEGRATION_LABEL=$1 + INTEGRATION_NAME=$2 + CALL=$3 + MAX_CHECK=${4:=1} ENV_VAR_NAME=INTEGRATION_${INTEGRATION_NAME^^} if [[ ${!ENV_VAR_NAME:=} != "true" ]]; then @@ -65,16 +69,16 @@ function check_integration() { fi return fi - check_service "${@}" + check_service "${INTEGRATION_LABEL}" "${CALL}" "${MAX_CHECK}" } -function check_db_backend() { +function check_db_backend { MAX_CHECK=${1:=1} if [[ ${BACKEND} == "postgres" ]]; then - check_service "postgres" "nc -zvv postgres 5432" "${MAX_CHECK}" + check_service "PostgreSQL" "nc -zvv postgres 5432" "${MAX_CHECK}" elif [[ ${BACKEND} == "mysql" ]]; then - check_service "mysql" "nc -zvv mysql 3306" "${MAX_CHECK}" + check_service "MySQL" "nc -zvv mysql 3306" "${MAX_CHECK}" elif [[ ${BACKEND} == "sqlite" ]]; then return else @@ -119,7 +123,8 @@ function startairflow_if_requested() { airflow initdb airflow create_user -u admin -p admin -f Thor -l Adminstra -r Admin -e [email protected] || true else - airflow create_user -u admin -p admin -f Thor -l Adminstra -r Admin -e [email protected] + airflow db init + airflow users create -u admin -p admin -f Thor -l Adminstra -r Admin -e [email protected] fi # shellcheck source=scripts/in_container/run_init_script.sh @@ -136,13 +141,14 @@ if [[ -n ${BACKEND=} ]]; then check_db_backend 20 echo "-----------------------------------------------------------------------------------------------" fi -check_integration kerberos "nc -zvv kerberos 88" 30 -check_integration mongo "nc -zvv mongo 27017" 20 -check_integration redis "nc -zvv redis 6379" 20 -check_integration rabbitmq "nc -zvv rabbitmq 5672" 20 -check_integration cassandra "nc -zvv cassandra 9042" 20 -check_integration openldap "nc -zvv openldap 389" 20 -check_integration presto "nc -zvv presto 8080" 40 +check_integration "Kerberos" "kerberos" "nc -zvv kerberos 88" 30 +check_integration "MongoDB" "mongo" "nc -zvv mongo 27017" 20 +check_integration "Redis" "redis" "nc -zvv redis 6379" 20 +check_integration "RabbitMQ" "rabbitmq" "nc -zvv rabbitmq 5672" 20 +check_integration "Cassandra" "cassandra" "nc -zvv cassandra 9042" 20 +check_integration "OpenLDAP" "openldap" "nc -zvv openldap 389" 20 +check_integration "Presto (API)" "presto" \ + "curl --max-time 1 http://presto:8080/v1/info/ | grep '\"starting\":false'" 20 echo "-----------------------------------------------------------------------------------------------" if [[ ${EXIT_CODE} != 0 ]]; then @@ -165,5 +171,3 @@ if [[ -n ${DISABLED_INTEGRATIONS=} ]]; then echo "Enable them via --integration <INTEGRATION_NAME> flags (you can use 'all' for all)" echo fi - -exit 0 diff --git a/scripts/in_container/configure_environment.sh b/scripts/in_container/configure_environment.sh index 3fe7858..01c3487 100644 --- a/scripts/in_container/configure_environment.sh +++ b/scripts/in_container/configure_environment.sh @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. # Script to check licences for all code. Can be started from any working directory -# shellcheck source=scripts/in_container/_in_container_script_init.sh export FILES_DIR="/files" export AIRFLOW_BREEZE_CONFIG_DIR="${FILES_DIR}/airflow-breeze-config" VARIABLES_ENV_FILE="variables.env" diff --git a/scripts/in_container/entrypoint_exec.sh b/scripts/in_container/entrypoint_exec.sh index 728872b..4423810 100755 --- a/scripts/in_container/entrypoint_exec.sh +++ b/scripts/in_container/entrypoint_exec.sh @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # shellcheck source=scripts/in_container/_in_container_script_init.sh . /opt/airflow/scripts/in_container/_in_container_script_init.sh diff --git a/tests/bats/breeze/test_breeze_complete.bats b/tests/bats/breeze/test_breeze_complete.bats index 2e9d1da..3ca32ce 100644 --- a/tests/bats/breeze/test_breeze_complete.bats +++ b/tests/bats/breeze/test_breeze_complete.bats @@ -272,3 +272,11 @@ assert_equal "" "${TEST_TYPE}" } + +@test "Test default package format is wheel" { + load ../bats_utils + #shellcheck source=breeze-complete + source "${AIRFLOW_SOURCES}/breeze-complete" + + assert_equal "wheel" "${PACKAGE_FORMAT}" +}
