This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch backport-ci-20261025-v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 40c7a841c38b7cdd233a217066913b0acd9bb594 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Oct 25 19:29:35 2026 +0100 Run maintainer and canary CI on self-hosted RunsOn runners (#74889) This PR implements AIP-118 for integrating self-hosted runners in Apache Airflow's CI, utilizing the project's AWS instance and runs-on.com's runners. (cherry picked from commit b5bc253fc2d594ccedb7da6104f2b03d26485651) Generated-by: Claude Opus 5 --- .github/actions/breeze/action.yml | 2 + .../actions/prepare_breeze_and_image/action.yml | 5 +- .github/actions/prepare_single_ci_image/action.yml | 5 +- .../actions/runs-on-cost-summary/action.yml | 32 +- .github/actions/stash-restore/action.yml | 137 +++++++ .github/actions/stash-save/action.yml | 129 +++++++ .github/workflows/additional-ci-image-checks.yml | 2 +- .github/workflows/additional-prod-image-tests.yml | 30 +- .github/workflows/basic-tests.yml | 26 +- .github/workflows/ci-amd.yml | 61 +-- .github/workflows/ci-arm.yml | 63 ++-- .github/workflows/ci-image-build.yml | 36 +- .github/workflows/ci-image-checks.yml | 8 +- .github/workflows/finalize-tests.yml | 6 +- .github/workflows/generate-constraints.yml | 4 +- .github/workflows/helm-tests.yml | 4 +- .github/workflows/integration-system-tests.yml | 6 +- .github/workflows/k8s-tests.yml | 4 +- .github/workflows/kustomize-overlays-tests.yml | 2 +- .github/workflows/prod-image-build.yml | 6 +- .github/workflows/push-image-cache.yml | 4 +- .github/workflows/special-tests.yml | 22 +- .github/workflows/test-providers.yml | 4 +- .github/workflows/ui-e2e-tests.yml | 2 +- README.md | 2 + dev/breeze/doc/ci/04_selective_checks.md | 90 ++++- dev/breeze/doc/ci/06_debugging.md | 6 +- dev/breeze/src/airflow_breeze/global_constants.py | 8 + .../src/airflow_breeze/utils/selective_checks.py | 116 ++++++ dev/breeze/tests/test_selective_checks.py | 411 +++++++++++++++++++++ scripts/ci/make_mnt_writeable.sh | 21 +- scripts/ci/move_docker_to_mnt.sh | 7 +- scripts/ci/prek/check_ci_workflows_in_sync.py | 5 +- 33 files changed, 1096 insertions(+), 170 deletions(-) diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index cb49d1d6d57..9fe5b551096 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -29,6 +29,8 @@ outputs: runs: using: "composite" steps: + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Setup python" uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6c91e28bff8..263db0346d4 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -82,12 +82,13 @@ runs: - name: > Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }} ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }} - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + uses: ./.github/actions/stash-restore with: key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ ${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}" path: "/mnt/" + # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image + # stashes as S3 digests only. only-current-branch: 'true' fail-on-download: 'true' - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml index d70012c80f7..8a734ef1102 100644 --- a/.github/actions/prepare_single_ci_image/action.yml +++ b/.github/actions/prepare_single_ci_image/action.yml @@ -36,11 +36,12 @@ runs: using: "composite" steps: - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}" - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + uses: ./.github/actions/stash-restore with: key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }} path: "/mnt/" + # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image + # stashes as S3 digests only. only-current-branch: 'true' fail-on-download: 'true' if: contains(inputs.python-versions-list-as-string, inputs.python) diff --git a/scripts/ci/make_mnt_writeable.sh b/.github/actions/runs-on-cost-summary/action.yml old mode 100755 new mode 100644 similarity index 58% copy from scripts/ci/make_mnt_writeable.sh copy to .github/actions/runs-on-cost-summary/action.yml index ddabf84b6b7..fd0d2007f10 --- a/scripts/ci/make_mnt_writeable.sh +++ b/.github/actions/runs-on-cost-summary/action.yml @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,20 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -function make_mnt_writeable { - set -x - echo "Investigating node disks" - lsblk - sudo blkid - echo "Checking free space!" - df -H - echo "Cleaning /mnt just in case it is not empty" - sudo mkdir -p /mnt - sudo rm -rf /mnt/* - echo "Checking free space!" - df -H - echo "Making sure that /mnt is writeable" - sudo chown -R "${USER}" /mnt -} - -make_mnt_writeable +# +--- +# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job +# that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze. +name: 'RunsOn cost summary' +description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.' +runs: + using: 'composite' + steps: + - name: "RunsOn cost summary" + if: runner.environment == 'self-hosted' + uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + with: + show_costs: summary diff --git a/.github/actions/stash-restore/action.yml b/.github/actions/stash-restore/action.yml new file mode 100644 index 00000000000..ed9535c1722 --- /dev/null +++ b/.github/actions/stash-restore/action.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On RunsOn (the only +# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's in-region S3 cache +# bucket instead of leaving AWS as GitHub artifacts. +# +# Every RunsOn job can write anywhere in that bucket, so the bucket is not trusted: objects are +# named by their sha256, and the digest itself travels as a small ASF stash artifact, which +# GitHub scopes to the run / branch exactly like a regular stash. A blob that does not match the +# digest the ASF restore found for this run, branch and repository is never extracted. The bucket's +# lifecycle rule (S3CacheExpirationInDays) must outlive the longest retention-days of a digest +# (6 days today), or a digest hit finds no blob. +name: 'Restore stash' +description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn runners.' +inputs: + key: + description: 'Stash key.' + required: true + path: + description: > + Directory to restore into (GitHub-hosted only; the S3 tarball restores entries to the + paths they were saved from, and the restore then fails if any of them is outside this directory). + required: false + default: ${{ github.workspace }} + only-current-branch: + description: 'Do not fall back to the base branch stash.' + required: false + default: 'false' + fail-on-download: + description: 'Fail when a stash was found but could not be restored.' + required: false + default: 'false' +outputs: + stash-hit: + description: "'true' if a stash was restored, 'false' otherwise." + value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit }} +runs: + using: 'composite' + steps: + - name: "Restore stash from GitHub artifacts" + id: artifact + if: runner.environment != 'self-hosted' + # yamllint disable-line rule:line-length + uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + with: + key: ${{ inputs.key }} + path: ${{ inputs.path }} + only-current-branch: ${{ inputs.only-current-branch }} + fail-on-download: ${{ inputs.fail-on-download }} + - name: "Restore stash digest from GitHub artifacts" + id: digest + if: runner.environment == 'self-hosted' + # yamllint disable-line rule:line-length + uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + with: + key: ${{ inputs.key }}-s3 + path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }} + only-current-branch: ${{ inputs.only-current-branch }} + fail-on-download: ${{ inputs.fail-on-download }} + - name: "Restore stash from S3" + id: s3 + if: runner.environment == 'self-hosted' + shell: bash + env: + STASH_KEY: ${{ inputs.key }} + STASH_PATH: ${{ inputs.path }} + DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }} + DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}/sha256 + FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }} + run: | + set -uo pipefail + payload="${RUNNER_TEMP}/stash-restore-$$.tar" + entries="${RUNNER_TEMP}/stash-restore-$$.entries" + miss() { + rm -f "${payload}" "${entries}" + echo "stash-hit=false" >> "${GITHUB_OUTPUT}" + if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true" ]]; then + echo "::error::$1" + exit 1 + fi + echo "$1" + exit 0 + } + if [[ "${DIGEST_HIT}" != "true" ]]; then + miss "No stash '${STASH_KEY}'" + fi + digest="$(cat "${DIGEST_FILE}")" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + miss "Malformed digest for stash '${STASH_KEY}'" + fi + bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}" + export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}" + object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar" + if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then + miss "Could not download ${object} for stash '${STASH_KEY}'" + fi + if ! echo "${digest} ${payload}" | sha256sum --check --status; then + miss "${object} does not match the digest recorded for stash '${STASH_KEY}'" + fi + # GNU tar detects the compression stash-save chose. + if ! tar -xPvf "${payload}" > "${entries}"; then + miss "Could not extract ${object} for stash '${STASH_KEY}'" + fi + rm -f "${payload}" + # The S3 tarball ignores `path`: fail loudly if a caller saves and restores different + # directories, which would work on GitHub-hosted runners only. + restore_dir="$(realpath -ms "${STASH_PATH}")" + restore_dir="${restore_dir%/}" + outside=() + while IFS= read -r entry; do + [[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}" + [[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}") + done < "${entries}" + rm -f "${entries}" + if [[ ${#outside[@]} -gt 0 ]]; then + printf '%s\n' "${outside[@]}" + echo "::error::Stash '${STASH_KEY}' restored the entries above outside of ${restore_dir}/" + exit 1 + fi + echo "stash-hit=true" >> "${GITHUB_OUTPUT}" + echo "Restored stash '${STASH_KEY}' from ${object}" diff --git a/.github/actions/stash-save/action.yml b/.github/actions/stash-save/action.yml new file mode 100644 index 00000000000..ecfd2409438 --- /dev/null +++ b/.github/actions/stash-save/action.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn the tarball goes +# to the stack's S3 cache bucket under its sha256 and only the digest is saved as an ASF stash +# artifact. See stash-restore for why. +name: 'Save stash' +description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.' +inputs: + key: + description: 'Stash key.' + required: true + path: + description: 'File, directory or glob to stash.' + required: true + if-no-files-found: + description: "'error' fails when path matches nothing, 'warn' or 'ignore' do not." + required: false + default: 'warn' + retention-days: + description: > + Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's + S3CacheExpirationInDays lifecycle rule expires the tarball itself. + required: false + default: '5' + compression-level: + description: "Zlib compression level (0-9). On RunsOn, '0' stores the tarball uncompressed." + required: false + default: '6' +runs: + using: 'composite' + steps: + - name: "Save stash to GitHub artifacts" + if: runner.environment != 'self-hosted' + uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + with: + key: ${{ inputs.key }} + path: ${{ inputs.path }} + if-no-files-found: ${{ inputs.if-no-files-found }} + retention-days: ${{ inputs.retention-days }} + compression-level: ${{ inputs.compression-level }} + - name: "Save stash to S3" + id: s3 + if: runner.environment == 'self-hosted' + shell: bash + env: + STASH_KEY: ${{ inputs.key }} + STASH_PATH: ${{ inputs.path }} + DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }} + IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }} + COMPRESSION_LEVEL: ${{ inputs.compression-level }} + run: | + set -euo pipefail + bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}" + export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}" + shopt -s nullglob + # shellcheck disable=SC2206 + candidates=( ${STASH_PATH} ) + existing=() + for p in "${candidates[@]}"; do + if [[ -e "${p}" ]]; then + existing+=("${p}") + fi + done + if [[ ${#existing[@]} -eq 0 ]]; then + if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then + echo "::error::No files found for path '${STASH_PATH}'" + exit 1 + fi + if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then + echo "::warning::No files found for path '${STASH_PATH}', skipping save" + fi + exit 0 + fi + # The tarball is streamed to S3 while being hashed: a multi-GB image never touches the + # runner's disk. Its key is only known at the end, so it lands under a temporary key and + # is copied server-side to its digest key. + prefix="s3://${bucket}/cache/airflow-stash" + upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar" + digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo" + digest_file="${RUNNER_TEMP}/stash-save-$$.sha256" + trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT + mkfifo "${digest_fifo}" + sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" & + digest_pid=$! + compression=() + if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then + compression=(-z) + fi + tar -cP "${compression[@]}" -f - "${existing[@]}" \ + | tee "${digest_fifo}" \ + | aws s3 cp --only-show-errors - "${upload}" + wait "${digest_pid}" + digest="$(cat "${digest_file}")" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Could not hash the stash '${STASH_KEY}'" + exit 1 + fi + object="${prefix}/sha256/${digest}.tar" + aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}" + # Leaving the upload behind only costs storage until the bucket's lifecycle rule expires it. + aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not remove ${upload}" + mkdir -p "${DIGEST_DIR}" + echo "${digest}" > "${DIGEST_DIR}/sha256" + echo "saved=true" >> "${GITHUB_OUTPUT}" + echo "Saved ${existing[*]} to ${object}" + - name: "Save stash digest to GitHub artifacts" + if: runner.environment == 'self-hosted' && steps.s3.outputs.saved == 'true' + uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + with: + key: ${{ inputs.key }}-s3 + path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index de26e37fbfc..7d60cf6ef41 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -62,7 +62,7 @@ jobs: # layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners. timeout-minutes: 35 name: Check that image builds quickly - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index dc5da4b211e..c850b783deb 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -152,7 +152,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of PROD image building" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -189,7 +189,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -220,7 +220,7 @@ jobs: task-sdk-integration-tests: timeout-minutes: 60 name: "Task SDK integration tests with PROD image" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -254,7 +254,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Regular e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -264,7 +264,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -276,7 +276,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Elasticsearch remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -288,7 +288,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "OpenSearch remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -300,7 +300,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "XCom object storage backend e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -311,7 +311,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Event driven e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -323,7 +323,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Java SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -335,7 +335,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Go SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -347,7 +347,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "TypeScript SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -359,7 +359,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "OpenLineage e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -377,7 +377,7 @@ jobs: compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} uses: ./.github/workflows/openlineage-e2e-compat-tests.yml with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -423,7 +423,7 @@ jobs: airflow-ctl-integration-tests: timeout-minutes: 60 name: "Airflow CTL integration tests with PROD image" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 4212f4bd872..88740c02c46 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -87,7 +87,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} steps: - name: "Cleanup repo" shell: bash @@ -105,7 +105,7 @@ jobs: run-breeze-integration-tests: timeout-minutes: 120 name: Breeze integration tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-breeze-integration-tests == 'true' steps: - name: "Cleanup repo" @@ -136,13 +136,15 @@ jobs: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. UV_PYTHON: "${{ inputs.default-python-version }}" SHARED_DISTRIBUTIONS: ${{ join(fromJSON(inputs.shared-distributions-as-json), ' ') }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 1 persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = "<X.Y.Z>"`. @@ -171,7 +173,7 @@ jobs: tests-scripts: timeout-minutes: 10 name: Scripts tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-scripts-tests == 'true' env: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. @@ -182,6 +184,8 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = "<X.Y.Z>"`. @@ -200,7 +204,7 @@ jobs: tests-ui: timeout-minutes: 15 name: React UI tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.run-ui-tests == 'true' steps: - name: "Cleanup repo" @@ -210,6 +214,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -237,7 +243,7 @@ jobs: check-translation-completness: timeout-minutes: 15 name: "Check translation completeness" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -254,7 +260,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -327,7 +333,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -381,7 +387,7 @@ jobs: test-airflow-standalone: timeout-minutes: 30 name: "Test Airflow standalone commands" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: AIRFLOW_HOME: ~/airflow FORCE_COLOR: 1 @@ -390,6 +396,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = "<X.Y.Z>"`. diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 9cf33000791..2de3f4f30ad 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -20,8 +20,9 @@ # `release-branch-ci.yml`. Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the # scheduled canary. The two workflows are physical copies of each other # (GitHub Actions has no cross-file YAML include), differing only in their -# trigger surface, the hardcoded `runner-type` / `platform` values in -# `build-info`'s outputs, and the architecture-specific print-platform job. +# trigger surface, the hardcoded `platform` value in `build-info`'s outputs +# (and the `PLATFORM` selective checks pick the runner type for), and the +# architecture-specific print-platform job. # Edit both files together when changing pipeline shape. name: Tests (AMD) on: # yamllint disable-line rule:truthy @@ -200,7 +201,7 @@ jobs: latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - # platform / runner-type are hardcoded per workflow file (this is the + # platform is hardcoded per workflow file (this is the # AMD copy). The matching ARM copy lives in ci-arm.yml. platform: "linux/amd64" postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} @@ -242,7 +243,7 @@ jobs: run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} - runner-type: '["ubuntu-22.04"]' + runner-type: ${{ steps.selective-checks.outputs.runner-type }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} @@ -324,6 +325,8 @@ jobs: COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + # Repository variable: "true" moves every run, pushes included, back to public runners. + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" PLATFORM: "linux/amd64" SELECTIVE_TESTS_ON_PUSH: "true" PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }} @@ -461,7 +464,7 @@ jobs: timeout-minutes: 45 name: "MyPy providers checks" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} if: needs.build-info.outputs.run-mypy-providers == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -502,7 +505,7 @@ jobs: timeout-minutes: 20 name: "Migration round-trip check" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.has-migrations == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -624,7 +627,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -653,7 +656,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -682,7 +685,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -712,7 +715,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -741,7 +744,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -772,7 +775,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -803,7 +806,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -833,7 +836,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -919,7 +922,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -949,7 +952,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -1060,7 +1063,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1076,7 +1079,7 @@ jobs: tests-go-sdk: name: "Go SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1092,6 +1095,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -1130,7 +1135,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1140,6 +1145,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -1164,7 +1171,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }} timeout-minutes: 15 permissions: contents: read @@ -1180,6 +1187,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup Java uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 with: @@ -1194,7 +1203,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 30 permissions: contents: read @@ -1226,7 +1235,7 @@ jobs: build-ts-sdk-docs: name: "TypeScript SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} timeout-minutes: 30 permissions: contents: read @@ -1263,7 +1272,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1284,7 +1293,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -1534,7 +1543,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" @@ -1544,6 +1553,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 4352ce5d05b..69ceccc2f3f 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -20,9 +20,10 @@ # `release-branch-ci.yml` — never on PRs. Companion to `ci-amd.yml`, which runs the # same pipeline on AMD for every PR. The two workflows are physical # copies of each other (GitHub Actions has no cross-file YAML include), -# differing only in their trigger surface, the hardcoded `runner-type` / -# `platform` values in `build-info`'s outputs, and the architecture-specific -# print-platform job. Edit both files together when changing pipeline shape. +# differing only in their trigger surface, the hardcoded `platform` value in +# `build-info`'s outputs (and the `PLATFORM` selective checks pick the runner +# type for), and the architecture-specific print-platform job. Edit both files +# together when changing pipeline shape. name: Tests (ARM) on: # yamllint disable-line rule:truthy schedule: @@ -179,7 +180,7 @@ jobs: latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - # platform / runner-type are hardcoded per workflow file (this is the + # platform is hardcoded per workflow file (this is the # ARM copy). The matching AMD copy lives in ci-amd.yml. platform: "linux/arm64" postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} @@ -221,7 +222,7 @@ jobs: run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} - runner-type: '["ubuntu-22.04-arm"]' + runner-type: ${{ steps.selective-checks.outputs.runner-type }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} @@ -303,6 +304,8 @@ jobs: COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + # Repository variable: "true" moves every run, pushes included, back to public runners. + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" PLATFORM: "linux/arm64" SELECTIVE_TESTS_ON_PUSH: "true" PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }} @@ -440,7 +443,7 @@ jobs: timeout-minutes: 45 name: "MyPy providers checks" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} if: needs.build-info.outputs.run-mypy-providers == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -481,7 +484,7 @@ jobs: timeout-minutes: 20 name: "Migration round-trip check" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.has-migrations == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -603,7 +606,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -632,7 +635,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -661,7 +664,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -691,7 +694,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -720,7 +723,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -751,7 +754,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -782,7 +785,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -812,7 +815,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -898,7 +901,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -928,7 +931,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -1039,7 +1042,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1055,7 +1058,7 @@ jobs: tests-go-sdk: name: "Go SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1071,6 +1074,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -1109,7 +1114,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1119,6 +1124,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -1143,7 +1150,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }} timeout-minutes: 15 permissions: contents: read @@ -1159,6 +1166,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup Java uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 with: @@ -1173,7 +1182,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 30 permissions: contents: read @@ -1205,7 +1214,7 @@ jobs: build-ts-sdk-docs: name: "TypeScript SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} timeout-minutes: 30 permissions: contents: read @@ -1242,7 +1251,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1263,7 +1272,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -1513,7 +1522,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" @@ -1523,6 +1532,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 315256420fd..23f8e1f724c 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -128,7 +128,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) }} timeout-minutes: 110 name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} @@ -147,24 +147,38 @@ jobs: with: ref: ${{ inputs.checkout-ref }} persist-credentials: false + # A local action is read from the workspace, and `checkout-ref` may be a release ref cut + # before the stash actions existed, so they are loaded from the workflow's own version. The + # runner-preparing scripts too: an older `make_mnt_writeable.sh` empties /mnt, which on + # RunsOn holds the runner's own disks. + - name: "Checkout the workflow version for its actions and runner scripts" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: workflow-version + sparse-checkout: | + .github/actions + scripts/ci - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh - name: "Make /mnt writeable" - run: ./scripts/ci/make_mnt_writeable.sh + run: ./workflow-version/scripts/ci/make_mnt_writeable.sh - name: "Move docker to /mnt" - run: ./scripts/ci/move_docker_to_mnt.sh + run: ./workflow-version/scripts/ci/move_docker_to_mnt.sh - name: "Install Breeze" uses: ./.github/actions/breeze # A ref is not a commit: a branch documented twice moves between the two publishes, and # an image built before it moved is a cache, not an answer. The commit each stashed image # is saved with tells the two apart, and it is stashed separately so that deciding costs # a few bytes rather than the multi-gigabyte image the decision may make unnecessary. + # `only-current-branch` must stay true on the image restores below: runs on RunsOn (canaries + # and release-branch pushes) save these image stashes as S3 digests only, so a fallback to the + # base branch would find nothing on public runners anyway. - name: > Restore the commit the CI image stashed for ref ${{ inputs.image-stash-ref }} was built from - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -197,8 +211,7 @@ jobs: - name: > Restore CI docker image built for ref ${{ inputs.image-stash-ref }} ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -244,8 +257,7 @@ jobs: --cache-file /tmp/ci-cache-mount-save-v4-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz if: steps.restore-cache-mount.outputs.stash-hit == 'true' - name: "Restore CI docker image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" path: "/mnt/" @@ -336,7 +348,7 @@ jobs: shell: bash if: inputs.upload-image-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" @@ -350,7 +362,7 @@ jobs: # what a later publish of that same ref reads - hence a retention that outlives the shared # one, the same ref being rebuilt three days apart at the very least. - name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -364,7 +376,7 @@ jobs: # Saved last and with the image's retention, so that finding this commit is enough to # know the image it describes is there to be restored. - name: "Stash the commit the CI image for ref ${{ inputs.image-stash-ref }} was built from" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index affe6ebdb8d..1a1c06c6804 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -119,7 +119,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -175,7 +175,7 @@ jobs: build-docs: timeout-minutes: 150 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} if: inputs.docs-build == 'true' env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -334,7 +334,7 @@ jobs: id-token: write contents: read needs: build-docs - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -466,7 +466,7 @@ jobs: test-python-api-client: timeout-minutes: 60 name: "Test Python API client" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-api-codegen == 'true' env: BACKEND: "postgres" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 043aa8f0ebf..b664e49cf06 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -103,6 +103,8 @@ jobs: with: # Needed to perform push action persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} @@ -124,7 +126,7 @@ jobs: # Constraint commits are now handled by the dedicated update-constraints-on-push workflow # triggered when uv.lock changes are pushed to main/test branches. dependency-upgrade-summary: - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} needs: [update-constraints] if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64' name: "Deps ${{ matrix.python-version }}:${{ matrix.constraints-mode }}" diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 9846e44ce71..58312ea63c4 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -85,7 +85,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints for ${{ matrix.python-version }} on ${{ inputs.platform }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -114,6 +114,8 @@ jobs: platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} + # Runs the checkout-ref's own make_mnt_writeable.sh, which on old refs empties /mnt and + # would kill a RunsOn runner: every caller passing checkout-ref pins public runners. make-mnt-writeable-and-cleanup: true - name: "Install prek" uses: ./.github/actions/install-prek diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 7e779afcc2a..2cbe0603a43 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -52,7 +52,7 @@ jobs: name: >- Unit tests Helm: ${{ matrix.helm-test-package }} (K8S ${{ matrix.kubernetes-version }}) - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} strategy: fail-fast: false max-parallel: 20 @@ -98,7 +98,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 202d70cd9ef..efc8d578b5c 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -75,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} strategy: fail-fast: false max-parallel: 20 @@ -126,7 +126,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true' name: "Integration: providers ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -176,7 +176,7 @@ jobs: timeout-minutes: 30 if: inputs.run-system-tests == 'true' name: "System Tests" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: BACKEND: "postgres" BACKEND_VERSION: ${{ inputs.default-postgres-version }} diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index d295b9d713d..39b0277e110 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -62,7 +62,7 @@ jobs: tests-kubernetes: timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -149,7 +149,7 @@ jobs: tests-kubernetes-lang-sdk: timeout-minutes: 60 name: "K8S Lang-SDK:${{ inputs.lang-sdk-kubernetes-combo }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} diff --git a/.github/workflows/kustomize-overlays-tests.yml b/.github/workflows/kustomize-overlays-tests.yml index 94637194bfa..e51def2487c 100644 --- a/.github/workflows/kustomize-overlays-tests.yml +++ b/.github/workflows/kustomize-overlays-tests.yml @@ -53,7 +53,7 @@ jobs: smoke-test-overlay: timeout-minutes: 40 name: "Overlay smoke test: ${{ matrix.overlay }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} strategy: fail-fast: false max-parallel: 4 diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index f136cd1ce61..31eaabace18 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -119,7 +119,7 @@ jobs: build-prod-packages: name: "Build Airflow and provider distributions" timeout-minutes: 10 - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -199,7 +199,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) }} timeout-minutes: 80 name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} needs: - build-prod-packages env: @@ -289,7 +289,7 @@ jobs: breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt" if: inputs.upload-image-artifact == 'true' - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./.github/actions/stash-save with: key: prod-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 4d04deb09da..3dc4693e0df 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -79,7 +79,7 @@ on: # yamllint disable-line rule:truthy jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} permissions: contents: read packages: write @@ -149,7 +149,7 @@ jobs: push-prod-image-cache: name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} permissions: contents: read packages: write diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 6ac4d60d1ba..ff56fbb9df5 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -94,7 +94,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -119,7 +119,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -144,7 +144,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -169,7 +169,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -194,7 +194,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -220,7 +220,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -246,7 +246,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" @@ -272,7 +272,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" @@ -298,7 +298,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" @@ -323,7 +323,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" @@ -348,7 +348,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index e7b3575b3f1..406ebde52fa 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -72,7 +72,7 @@ jobs: prepare-install-verify-provider-distributions: timeout-minutes: 80 name: "Providers ${{ matrix.package-format }} tests" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false matrix: @@ -178,7 +178,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.test-types.description }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index 8b8ad21e4bf..1ee5e1e6a67 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -88,7 +88,7 @@ jobs: test-ui-e2e-tests: timeout-minutes: 90 name: ${{ inputs.workflow-name || 'UI E2E Tests' }} - runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }} + runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11' }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/README.md b/README.md index 7a76cb4dc00..e0e77777ab4 100644 --- a/README.md +++ b/README.md @@ -603,3 +603,5 @@ The CI infrastructure for Apache Airflow has been sponsored by: <a href="https://astronomer.io";><img src="https://assets2.astronomer.io/logos/logoForLIGHTbackground.png"; alt="astronomer.io" width="250px"></a> <a href="https://aws.amazon.com/opensource/";><img src="https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/[email protected]?raw=true"; alt="AWS OpenSource" width="130px"></a> + +The self-hosted CI runners on AWS are managed by [RunsOn](https://runs-on.com), which is free for open-source projects. diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 7b383f95566..45937cf3b71 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -33,7 +33,7 @@ - [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks) - [Suspended providers](#suspended-providers) - [Selective check outputs](#selective-check-outputs) - - [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs) + - [Self-hosted (RunsOn) vs. public runners](#self-hosted-runson-vs-public-runners) - [Changing behaviours of the CI runs by setting labels](#changing-behaviours-of-the-ci-runs-by-setting-labels) <!-- END doctoc generated TOC please keep comment here to allow auto update --> @@ -664,6 +664,7 @@ GitHub Actions to pass the list of parameters to a command to execute | run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | | amd-runners | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | | | arm-runners | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | | +| runner-type | Runner labels for the CI jobs on this platform: public, or a RunsOn template jobs fill with cpu/ram | \["ubuntu-22.04"\] | | | selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | | skip-prek-hooks | Which prek hooks should be skipped during the static-checks run | flynt,identity | | | skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | @@ -673,24 +674,79 @@ GitHub Actions to pass the list of parameters to a command to execute | upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | -## Committer vs. Non-committer PRs - -There is a difference in how the CI jobs are run for committer and non-committer PRs from forks. -The main reason is security; we do not want to run untrusted code on our infrastructure for self-hosted runners. - -Currently there is no difference because we are not using `self-hosted` runners (until we implement `Action -Runner Controller` but most of the jobs, committer builds will use "Self-hosted" runners by default, -while non-committer builds will use "Public" runners. For committers, this can be overridden by setting the -`use public runners` label in the PR. +## Self-hosted (RunsOn) vs. public runners + +The `Tests (AMD)` / `Tests (ARM)` jobs run either on GitHub-hosted ("public") runners or on self-hosted +[RunsOn](https://runs-on.com) EC2 runners in the Airflow AWS account. Selective checks route runs of +code from `apache/airflow` itself there; this is a routing preference, not what keeps fork code off +RunsOn (see below). The `runner-type` output is decided as follows, first match wins: + +1. Runs in any repository other than `apache/airflow` (forks running their own CI) use public runners. +2. Any run uses public runners while the repository variable `USE_PUBLIC_RUNNERS` is `true`: the + switch for moving `main` off RunsOn without a PR, e.g. when the stack is unavailable. +3. PRs with the `use public runners` label use public runners. +4. Any `push`, `schedule` or `workflow_dispatch` in `apache/airflow` (canary runs and the release + branch pushes, all triggered by people with write access) uses RunsOn. +5. Other events than `pull_request` use public runners. +6. PRs with the `use self-hosted runners` label use RunsOn, if a committer's (`COMMITTERS` in + `global_constants.py`) latest review of the PR's head commit is an approval and no committer's + request for changes stands. The label outlives the commit it was added for, so on its own it would + vouch for every later push: a new push sends the PR back to public runners until the head commit is + approved again, and so does a dismissed approval if no other committer's approval of the head commit + remains. A committer's request for changes keeps the PR on public runners across pushes, as it blocks + a merge on GitHub, until that committer reviews again or the review is dismissed. The label must + also still be on the PR: a re-run reuses the labels of its original event, so they are checked + against the PR's current ones. If the PR or its reviews cannot be read, the label is ignored as well. +7. PRs with the `non committer build` label use public runners. +8. PRs opened by a bot (Dependabot and the like), or from a `dependabot/` branch, use public runners: + they carry code nobody has reviewed yet. +9. PRs from a branch of `apache/airflow` use RunsOn; PRs from forks use public runners. + +Labels are read from the event that started the run, and the CI workflows do not run on `labeled`. After +adding or removing a label, push a commit or close and reopen the PR; re-running an earlier run reuses +its original labels, and "Re-run failed jobs" also reuses that run's runner decision. A push voids the +approval `use self-hosted runners` needs, so for that label approve the head commit and then close and +reopen the PR. + +The RunsOn value is a label template such as +`["runs-on=<run_id>/family=.../image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]`. Every job +picks its size class with `format(fromJSON(inputs.runners)[0], <cpu>, <ram>)` (4/16, 8/32 or 32/128), +or, for a single-job reusable workflow, its caller passes `format(<runners>, <cpu>, <ram>)`; +on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are +gated on `runner.environment == 'self-hosted'`: + +* The EC2 cost summary of each job. +* The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to + the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact. + Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by + the same run, branch and repository (or, for a PR, its base branch). The smaller caches (mount cache, prek, docs inventory) use + the ASF stash action on both kinds of runners, so PRs on public runners keep the ones canary runs save; + the price is that every RunsOn canary uploads the mount cache from AWS to GitHub. + +The decision above is not a security boundary. Under the `pull_request` event the code that +evaluates it is the PR's own: a fork PR runs its own workflow files and breeze sources, and can name +RunsOn labels in `runs-on:` directly. RunsOn launches a runner for any job of a repository its GitHub +App is installed on, so nothing in this repository can prevent that. Keeping untrusted code away from +anything that matters has to be enforced outside the repository, by: + +* The repository's Actions setting "Approval for running fork pull request workflows from + contributors", which must be "Require approval for all external contributors". That still exempts + members of the `apache` GitHub organization (ASF committers of any project who linked their GitHub + account) and repository collaborators, but every exempt person is identifiable. +* The RunsOn stack, which must be dedicated to this repository and give the runners an instance role + that reaches nothing beyond the stack's own cache bucket (whose contents are never trusted, see the + stash actions), no EBS permissions, no SSH, network isolation between runners and a maximum run + time. RunsOn has no cap on instance sizes or counts and its budget only alerts, so the AWS account's + EC2 vCPU quotas are the cost ceiling. ## Changing behaviours of the CI runs by setting labels -Also, currently for most of the jobs, committer builds by default use "Self-hosted" runners, while -non-committer builds use "Public" runners. For committers, this can be overridden by setting the -`use public runners` label in the PR. In the future when we might also switch committers to public runners. -Committers will be able to use `use self-hosted runners` label in the PR to force using self-hosted runners. -The `use public runners` label will still be available for committers and they will be able to set it for -builds that also have `canary` label set to also switch the `canary` builds to public runners. +PRs from `apache/airflow` branches run on self-hosted RunsOn runners and PRs from forks run on public +runners (see above). Set `use public runners` to move a PR to public runners; a committer can set +`use self-hosted runners` and approve the PR's head commit to move a fork PR to RunsOn, provided no +committer's request for changes stands, then close and reopen the PR (a push would need a fresh +approval; a committer should do the reopen, since an external author's reopen still needs a maintainer +to approve the workflow run). If you are testing CI workflow changes and want to test it for more complete matrix combinations generated by the jobs - you can set `all versions` label in the PR. This will run the PRs with the same combinations @@ -698,7 +754,7 @@ of versions as the `canary` main build. Using `all versions` is automatically se change in `pyproject.toml`. If you are running an `apache` PR, you can also set `canary` label for such PR and in this case, all the -`canary` properties of build will be used: `self-hosted` runners, `full tests needed` mode, `all versions` +`canary` properties of build will be used: `full tests needed` mode, `all versions` as well as all canary-specific jobs will run there. You can modify this behaviour of the `canary` run by applying `use public runners`, and `default versions only` labels to the PR as well which will still run a `canary` equivalent build but with public runners an default Python/K8S versions only - respectively. diff --git a/dev/breeze/doc/ci/06_debugging.md b/dev/breeze/doc/ci/06_debugging.md index c87c2b3f5b0..dfe9b968931 100644 --- a/dev/breeze/doc/ci/06_debugging.md +++ b/dev/breeze/doc/ci/06_debugging.md @@ -44,12 +44,16 @@ in some cases, you need to run the PR as coming from the "apache" repository rather than from your fork. You can also apply the label later and rebase the PR or close/reopen -the PR to apply the label to the PR. +the PR to apply the label to the PR. For `use self-hosted runners`, a +committer approves the head commit (with no committer's request for +changes standing) and closes/reopens the PR: a rebase needs a fresh +approval. | Action to perform | Label to set | PR from "apache" repo | |------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|:---------------------:| | Run the build with all combinations of all<br>python, backends, kubernetes etc on PR, <br>and run all types of tests for all test<br>groups. | full tests needed | | | Force to use public runners for the build | use public runners | | +| Run a PR from a fork on the self-hosted RunsOn<br>runners (needs a committer's approval of its head commit<br>and no committer's request for changes) | use self-hosted runners | | | Debug resources used during the build for <br>parallel jobs | debug ci resources | | | Force running PR on latest versions of<br>python, backends, kubernetes etc. when you<br>want to save resources and test only latest<br>versions | latest versions only | | | Force running PR on minimal (default) <br>versions of python, backends, kubernetes etc.<br>in order to save resources and run tests only<br>for minimum versions | default versions only | | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 60615aa1c66..0d4c79793ff 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -38,6 +38,14 @@ from airflow_breeze.utils.path_utils import ( PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]' PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]' +# RunsOn (runs-on.com) EC2 runner label templates. Jobs fill in their size class with +# `format(<label>, cpu, ram)` in the workflow; the same call leaves the public labels unchanged. +RUNS_ON_AMD_RUNNER_TEMPLATE = ( + "runs-on={run_id}/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={{0}}/ram={{1}}" +) +RUNS_ON_ARM_RUNNER_TEMPLATE = ( + "runs-on={run_id}/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={{0}}/ram={{1}}" +) # Platform the tests of a CI run execute on. CI_AMD_PLATFORM = "linux/amd64" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index e441fa7149f..d272587c11e 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -55,6 +55,8 @@ from airflow_breeze.global_constants import ( PROVIDERS_COMPATIBILITY_TESTS_MATRIX, PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS, + RUNS_ON_AMD_RUNNER_TEMPLATE, + RUNS_ON_ARM_RUNNER_TEMPLATE, TESTABLE_CORE_INTEGRATIONS, TESTABLE_PROVIDERS_INTEGRATION_OWNERS, TESTABLE_PROVIDERS_INTEGRATIONS, @@ -94,6 +96,7 @@ LATEST_VERSIONS_ONLY_LABEL = "latest versions only" NON_COMMITTER_BUILD_LABEL = "non committer build" UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies" USE_PUBLIC_RUNNERS_LABEL = "use public runners" +USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners" ALLOW_PROVIDER_DEPENDENCY_BUMP_LABEL = "allow provider dependency bump" SKIP_COMMON_COMPAT_CHECK_LABEL = "skip common compat check" AREA_E2E_TESTS_LABEL = "area:e2e-tests" @@ -2138,6 +2141,119 @@ class SelectiveChecks: def arm_runners(self) -> str: return PUBLIC_ARM_RUNNERS + @cached_property + def runner_type(self) -> str: + """Runner labels for the jobs of a CI run on this run's platform. + + The RunsOn label is a template the jobs fill in with their size class. + """ + # RunsOn needs the run id in the label; without one (e.g. a local run) the jobs would never start. + run_id = self._github_context_dict.get("run_id") + if not run_id or not self._use_runs_on_runners(): + return PUBLIC_ARM_RUNNERS if self._platform == CI_ARM_PLATFORM else PUBLIC_AMD_RUNNERS + template = ( + RUNS_ON_ARM_RUNNER_TEMPLATE if self._platform == CI_ARM_PLATFORM else RUNS_ON_AMD_RUNNER_TEMPLATE + ) + return json.dumps([template.format(run_id=run_id)]) + + def _use_runs_on_runners(self) -> bool: + """Whether this run should use the self-hosted RunsOn runners. + + This only routes runs; it does not keep fork code off RunsOn. A ``pull_request`` run uses + the PR's own workflow files and breeze, so a fork can request a RunsOn label directly - + that restriction has to be enforced by the RunsOn stack and the repository's Actions + settings, not here. Routed there: pushes, schedules and dispatches, and PRs from + apache/airflow branches except bots' (e.g. Dependabot's). A fork PR goes there while it + carries the ``use self-hosted runners`` label, a committer has approved its head commit and + no committer's request for changes is outstanding. + The repository variable ``USE_PUBLIC_RUNNERS=true`` moves every run back to public runners. + """ + if self._github_repository != APACHE_AIRFLOW_GITHUB_REPOSITORY: + return False + if os.environ.get("USE_PUBLIC_RUNNERS", "").casefold() == "true": + return False + if USE_PUBLIC_RUNNERS_LABEL in self._pr_labels: + return False + if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE, GithubEvents.WORKFLOW_DISPATCH]: + return True + if self._github_event != GithubEvents.PULL_REQUEST: + return False + if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels and self._is_self_hosted_label_approved(): + return True + if NON_COMMITTER_BUILD_LABEL in self._pr_labels: + return False + pull_request = self._github_context_dict.get("event", {}).get("pull_request", {}) + # The PR author, not github.actor: a committer rebasing a Dependabot PR becomes the actor. + # The branch too: anyone can open their own PR from a Dependabot branch. + if (pull_request.get("user") or {}).get("type") == "Bot" or ( + pull_request.get("head", {}).get("ref", "").startswith("dependabot/") + ): + return False + # A PR from a deleted fork has no head repo. + head_repo = (pull_request.get("head", {}).get("repo") or {}).get("full_name") + return head_repo == APACHE_AIRFLOW_GITHUB_REPOSITORY + + def _is_self_hosted_label_approved(self) -> bool: + """Whether the PR still has the label, a committer approved its head commit and no committer's veto stands. + + A re-run reuses the labels of its original event, so a later removal only shows in the PR + itself. The label outlives the commit it was added for, so on its own it would vouch for + every later push. A request for changes stays in force across pushes, as on GitHub, until + that committer reviews again or it is dismissed. Any failure to tell counts as "no". + """ + # Lazy: breeze keeps its top-level imports minimal (check-breeze-top-dependencies-limited). + import requests + + pull_request = self._github_context_dict.get("event", {}).get("pull_request", {}) + head_sha = pull_request.get("head", {}).get("sha") + if not head_sha: + return False + pull_request_url = ( + f"https://api.github.com/repos/{self._github_repository}/pulls/{pull_request.get('number')}" + ) + headers = {"Accept": "application/vnd.github+json"} + if token := os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {token}" + # Each reviewer's latest verdict, on the head commit and on any commit; a dismissed review is + # listed as DISMISSED in place, and comment-only reviews change nothing. + latest_on_head: dict[str, str] = {} + latest_overall: dict[str, str] = {} + try: + response = requests.get(pull_request_url, headers=headers, timeout=30) + response.raise_for_status() + if USE_SELF_HOSTED_RUNNERS_LABEL not in {label["name"] for label in response.json()["labels"]}: + console_print( + f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r} label: the PR no longer has it" + ) + return False + url: str | None = f"{pull_request_url}/reviews?per_page=100" + while url: + response = requests.get(url, headers=headers, timeout=30) + response.raise_for_status() + for review in response.json(): + if review.get("state") not in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}: + continue + login = ((review.get("user") or {}).get("login") or "").casefold() + latest_overall[login] = review["state"] + if review.get("commit_id") == head_sha: + latest_on_head[login] = review["state"] + url = response.links.get("next", {}).get("url") + # AttributeError / KeyError / TypeError: a 200 whose body is not the expected PR or list of reviews. + except (requests.RequestException, AttributeError, KeyError, TypeError) as e: + console_print(f"[warning]Could not read the labels and reviews of the PR: {e}") + return False + committers = {committer.casefold() for committer in COMMITTERS} + approved = any(latest_on_head.get(committer) == "APPROVED" for committer in committers) + vetoed = any(latest_overall.get(committer) == "CHANGES_REQUESTED" for committer in committers) + if approved and not vetoed: + return True + console_print( + f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r} label: " + f"the head commit {head_sha} is not approved by a committer, or a committer's request for " + "changes stands" + ) + return False + @cached_property def has_migrations(self) -> bool: return any([file.startswith("airflow-core/src/airflow/migrations/") for file in self._files]) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 26bc452fd24..174012ebaef 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -23,6 +23,7 @@ from typing import Any from unittest.mock import Mock, patch import pytest +import requests from rich.console import Console from airflow_breeze.global_constants import ( @@ -30,6 +31,7 @@ from airflow_breeze.global_constants import ( ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, CI_AMD_PLATFORM, CI_ARM_PLATFORM, + COMMITTERS, CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, DEFAULT_KUBERNETES_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, @@ -4785,3 +4787,412 @@ def test_providers_with_test_side_effects_are_shared_process_providers(): for provider in PROVIDERS_WITH_PROCESS_GLOBAL_TEST_SIDE_EFFECTS: assert provider in get_available_distributions() assert provider not in LONG_RUNNING_TEST_PROVIDERS + + +RUNS_ON_AMD = '["runs-on=42/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]' +RUNS_ON_ARM = '["runs-on=42/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={0}/ram={1}"]' + + +HEAD_SHA = "a" * 40 + + [email protected](autouse=True) +def _no_use_public_runners_variable(monkeypatch): + monkeypatch.delenv("USE_PUBLIC_RUNNERS", raising=False) + + +def _pull_request_context(head_repo: str | None, author_type: str = "User") -> dict[str, Any]: + head = {"repo": {"full_name": head_repo} if head_repo else None, "sha": HEAD_SHA} + return { + "run_id": 42, + "event": {"pull_request": {"number": 7, "head": head, "user": {"type": author_type}}}, + } + + [email protected]( + ("github_event", "github_repository", "github_context_dict", "pr_labels", "platform", "expected"), + [ + pytest.param( + GithubEvents.PUSH, + "apache/airflow", + {"run_id": 42}, + (), + CI_AMD_PLATFORM, + RUNS_ON_AMD, + id="canary push uses RunsOn", + ), + pytest.param( + GithubEvents.SCHEDULE, + "apache/airflow", + {"run_id": 42}, + (), + CI_ARM_PLATFORM, + RUNS_ON_ARM, + id="scheduled ARM canary uses ARM RunsOn", + ), + pytest.param( + GithubEvents.WORKFLOW_DISPATCH, + "apache/airflow", + {"run_id": 42}, + (), + CI_AMD_PLATFORM, + RUNS_ON_AMD, + id="workflow_dispatch uses RunsOn", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("apache/airflow", author_type="Bot"), + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="bot PR from an apache/airflow branch uses public runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + { + "run_id": 42, + "event": { + "pull_request": { + "head": {"ref": "dependabot/uv/foo", "repo": {"full_name": "apache/airflow"}}, + "user": {"type": "User"}, + } + }, + }, + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="user PR from a Dependabot branch uses public runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("apache/airflow", author_type="Bot"), + ("use self-hosted runners", "non committer build"), + CI_AMD_PLATFORM, + RUNS_ON_AMD, + id="committer's self-hosted label wins over bot author and non committer build", + ), + pytest.param( + GithubEvents.PUSH, + "apache/airflow", + {}, + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="push without a run id stays on public runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("someone/airflow"), + ("use self-hosted runners", "use public runners"), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="public runners label wins over the self-hosted label", + ), + pytest.param( + GithubEvents.PUSH, + "someone/airflow", + {"run_id": 42}, + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="push in a fork repository stays on public runners", + ), + pytest.param( + GithubEvents.SCHEDULE, + "someone/airflow", + {"run_id": 42}, + (), + CI_ARM_PLATFORM, + '["ubuntu-22.04-arm"]', + id="schedule in a fork repository stays on public ARM runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("apache/airflow"), + (), + CI_AMD_PLATFORM, + RUNS_ON_AMD, + id="PR from an apache/airflow branch uses RunsOn", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("someone/airflow"), + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="PR from a fork uses public runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context(None), + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="PR from a deleted fork uses public runners", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("someone/airflow"), + ("use self-hosted runners",), + CI_AMD_PLATFORM, + RUNS_ON_AMD, + id="PR from a fork with the self-hosted label uses RunsOn", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("someone/airflow"), + ("canary", "full tests needed"), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="canary label does not route a fork PR to RunsOn", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("apache/airflow"), + ("use public runners",), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="public runners label opts an apache/airflow PR out", + ), + pytest.param( + GithubEvents.PULL_REQUEST, + "apache/airflow", + _pull_request_context("apache/airflow"), + ("non committer build",), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="non committer build label treats an apache/airflow PR as a fork PR", + ), + pytest.param( + GithubEvents.PULL_REQUEST_TARGET, + "apache/airflow", + _pull_request_context("someone/airflow"), + (), + CI_AMD_PLATFORM, + '["ubuntu-22.04"]', + id="pull_request_target uses public runners", + ), + ], +) [email protected](SelectiveChecks, "_is_self_hosted_label_approved", autospec=True, return_value=True) +def test_runner_type( + _mock_is_self_hosted_label_approved, + github_event: GithubEvents, + github_repository: str, + github_context_dict: dict[str, Any], + pr_labels: tuple[str, ...], + platform: str, + expected: str, +): + selective_checks = SelectiveChecks( + github_event=github_event, + github_repository=github_repository, + github_context_dict=github_context_dict, + pr_labels=pr_labels, + platform=platform, + ) + assert selective_checks.runner_type == expected + + [email protected]( + ("value", "expected"), + [ + pytest.param("true", '["ubuntu-22.04"]', id="true moves a push to public runners"), + pytest.param("True", '["ubuntu-22.04"]', id="case does not matter"), + pytest.param("false", RUNS_ON_AMD, id="false keeps RunsOn"), + pytest.param("", RUNS_ON_AMD, id="an unset variable keeps RunsOn"), + ], +) +def test_use_public_runners_variable(monkeypatch, value: str, expected: str): + monkeypatch.setenv("USE_PUBLIC_RUNNERS", value) + selective_checks = SelectiveChecks( + github_event=GithubEvents.PUSH, github_repository="apache/airflow", github_context_dict={"run_id": 42} + ) + assert selective_checks.runner_type == expected + + [email protected](SelectiveChecks, "_is_self_hosted_label_approved", autospec=True, return_value=True) +def test_use_public_runners_variable_beats_the_self_hosted_label( + _mock_is_self_hosted_label_approved, monkeypatch +): + monkeypatch.setenv("USE_PUBLIC_RUNNERS", "true") + selective_checks = SelectiveChecks( + github_event=GithubEvents.PULL_REQUEST, + github_repository="apache/airflow", + github_context_dict=_pull_request_context("apache/airflow"), + pr_labels=("use self-hosted runners",), + ) + assert selective_checks.runner_type == '["ubuntu-22.04"]' + + +def _review(login: str, state: str, commit_id: str = HEAD_SHA) -> dict[str, Any]: + return {"user": {"login": login}, "state": state, "commit_id": commit_id} + + [email protected]( + ("pages", "expected"), + [ + pytest.param([[_review(COMMITTERS[0], "APPROVED")]], True, id="approved by a committer"), + pytest.param( + [[_review(COMMITTERS[0].swapcase(), "APPROVED")]], + True, + id="approved by a committer whose login differs in case", + ), + pytest.param([[_review("a-triager", "APPROVED")]], False, id="approved by a non-committer"), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED", commit_id="b" * 40)]], + False, + id="approved an earlier commit only", + ), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "CHANGES_REQUESTED")]], + False, + id="changes requested after approving", + ), + pytest.param( + [[_review(COMMITTERS[0], "CHANGES_REQUESTED"), _review(COMMITTERS[0], "APPROVED")]], + True, + id="approved after requesting changes", + ), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "DISMISSED")]], + False, + id="approval dismissed", + ), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "COMMENTED")]], + True, + id="a later comment keeps the approval", + ), + pytest.param( + [[_review("a-triager", "APPROVED")], [_review(COMMITTERS[0], "APPROVED")]], + True, + id="approved on a later page", + ), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[1], "CHANGES_REQUESTED")]], + False, + id="another committer's request for changes vetoes an approval", + ), + pytest.param( + [[_review(COMMITTERS[0], "APPROVED"), _review("a-triager", "CHANGES_REQUESTED")]], + True, + id="a non-committer's request for changes does not veto", + ), + pytest.param( + [ + [ + _review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b" * 40), + _review(COMMITTERS[0], "APPROVED"), + ] + ], + False, + id="a committer's request for changes on an earlier commit still vetoes", + ), + pytest.param( + [ + [ + _review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b" * 40), + _review(COMMITTERS[1], "APPROVED", commit_id="b" * 40), + _review(COMMITTERS[0], "APPROVED"), + ] + ], + True, + id="a request for changes withdrawn by its committer's later approval does not veto", + ), + pytest.param([[]], False, id="no review"), + pytest.param([{"message": "Not Found"}], False, id="error object instead of reviews"), + ], +) +@patch("requests.get", autospec=True) +def test_is_self_hosted_label_approved(mock_get, pages: list[list[dict[str, Any]]], expected: bool): + pull_request = Mock(spec=requests.Response) + pull_request.json.return_value = {"labels": [{"name": "use self-hosted runners"}]} + responses = [pull_request] + for index, page in enumerate(pages): + response = Mock(spec=requests.Response) + response.json.return_value = page + response.links = {"next": {"url": f"page-{index + 1}"}} if index + 1 < len(pages) else {} + responses.append(response) + mock_get.side_effect = responses + selective_checks = SelectiveChecks( + github_context_dict=_pull_request_context("someone/airflow"), + pr_labels=("use self-hosted runners",), + ) + assert selective_checks._is_self_hosted_label_approved() is expected + assert [call.args[0] for call in mock_get.call_args_list[:2]] == [ + "https://api.github.com/repos/apache/airflow/pulls/7";, + "https://api.github.com/repos/apache/airflow/pulls/7/reviews?per_page=100";, + ] + + [email protected]( + "pull_request_body", + [ + pytest.param({"labels": [{"name": "full tests needed"}]}, id="label removed since the event"), + pytest.param({"message": "Not Found"}, id="error object instead of the PR"), + ], +) +@patch("requests.get", autospec=True) +def test_is_self_hosted_label_approved_needs_the_current_label(mock_get, pull_request_body: dict[str, Any]): + # A re-run reuses the labels of its original event, so the label is still in pr_labels. + response = Mock(spec=requests.Response) + response.json.return_value = pull_request_body + mock_get.return_value = response + selective_checks = SelectiveChecks( + github_context_dict=_pull_request_context("someone/airflow"), + pr_labels=("use self-hosted runners",), + ) + assert selective_checks._is_self_hosted_label_approved() is False + mock_get.assert_called_once() + + +@patch("requests.get", autospec=True) +def test_is_self_hosted_label_approved_without_head_sha(mock_get): + # A review whose commit_id is missing must not match a missing head sha. + response = Mock(spec=requests.Response) + response.json.return_value = [{"user": {"login": COMMITTERS[0]}, "state": "APPROVED"}] + response.links = {} + mock_get.return_value = response + context = _pull_request_context("someone/airflow") + del context["event"]["pull_request"]["head"]["sha"] + selective_checks = SelectiveChecks(github_context_dict=context, pr_labels=("use self-hosted runners",)) + assert selective_checks._is_self_hosted_label_approved() is False + mock_get.assert_not_called() + + [email protected]( + ("head_repo", "expected"), + [ + pytest.param("someone/airflow", '["ubuntu-22.04"]', id="fork PR falls back to public runners"), + pytest.param("apache/airflow", RUNS_ON_AMD, id="apache/airflow PR stays on RunsOn"), + ], +) [email protected]( + "failure", + [ + pytest.param(requests.ConnectionError("down"), id="connection error"), + pytest.param(requests.HTTPError("403"), id="non-2xx response"), + ], +) +@patch("requests.get", autospec=True) +def test_self_hosted_label_ignored_when_reviews_cannot_be_read( + mock_get, failure: Exception, head_repo: str, expected: str +): + mock_get.side_effect = failure + selective_checks = SelectiveChecks( + github_event=GithubEvents.PULL_REQUEST, + github_context_dict=_pull_request_context(head_repo), + pr_labels=("use self-hosted runners",), + ) + assert selective_checks.runner_type == expected diff --git a/scripts/ci/make_mnt_writeable.sh b/scripts/ci/make_mnt_writeable.sh index ddabf84b6b7..ed1e4d044ce 100755 --- a/scripts/ci/make_mnt_writeable.sh +++ b/scripts/ci/make_mnt_writeable.sh @@ -22,13 +22,26 @@ function make_mnt_writeable { sudo blkid echo "Checking free space!" df -H - echo "Cleaning /mnt just in case it is not empty" + echo "Cleaning /mnt just in case it is not empty and making sure that /mnt is writeable" sudo mkdir -p /mnt - sudo rm -rf /mnt/* + if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then + # RunsOn mounts an instance's local NVMe disks under /mnt and keeps docker, /home/runner and + # /tmp on them, so emptying or re-owning such a mount would break the running job. + local entry + for entry in /mnt/*; do + if mountpoint -q "${entry}"; then + echo "Keeping mount point ${entry}" + else + sudo rm -rf --one-file-system "${entry}" + fi + done + sudo chown "${USER}" /mnt + else + sudo rm -rf /mnt/* + sudo chown -R "${USER}" /mnt + fi echo "Checking free space!" df -H - echo "Making sure that /mnt is writeable" - sudo chown -R "${USER}" /mnt } make_mnt_writeable diff --git a/scripts/ci/move_docker_to_mnt.sh b/scripts/ci/move_docker_to_mnt.sh index e3d69ef60be..4fa5e4a0711 100755 --- a/scripts/ci/move_docker_to_mnt.sh +++ b/scripts/ci/move_docker_to_mnt.sh @@ -29,7 +29,12 @@ function cleanup_runner { # Hence we only move docker to /mnt on x86_64 where we have a separate /mnt mount # If we get short on disk space on arm64 as well we need to revisit this logic # and make the idle nvme being used as well. - if uname -i|grep -q x86_64; then + if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then + # RunsOn keeps docker either on the instance's local NVMe disks, which may also back + # /home/runner and /tmp, or - on instance types without them - on the same EBS root + # volume /mnt is on. Moving it gains nothing and could delete the running job's files. + echo "Self-hosted runner - keeping /var/lib/docker where it is" + elif uname -i|grep -q x86_64; then local target_docker_volume_location="/mnt/var-lib-docker" # This is faster than docker prune echo "Stopping docker" diff --git a/scripts/ci/prek/check_ci_workflows_in_sync.py b/scripts/ci/prek/check_ci_workflows_in_sync.py index 346aaa61926..933c6a5dc2e 100755 --- a/scripts/ci/prek/check_ci_workflows_in_sync.py +++ b/scripts/ci/prek/check_ci_workflows_in_sync.py @@ -34,8 +34,8 @@ Documented divergences: branches. Only AMD runs on PRs and pushes (ARM coverage stays cron-driven via the canary and the dispatched release-branch runs). 4. ``concurrency.group`` prefix — ``ci-arm-`` vs ``ci-amd-`` -5. ``build-info`` outputs ``platform`` and ``runner-type`` — hardcoded per - architecture (and the surrounding comment naming the "ARM/AMD copy") +5. ``build-info`` output ``platform`` — hardcoded per architecture (and the + surrounding comment naming the "ARM/AMD copy") 6. ``PLATFORM`` env on the ``selective-checks`` step — the platform this workflow's tests run on. 7. ``print-platform`` job — ``name:`` and the architecture echoed to @@ -89,7 +89,6 @@ LINE_RULES: list[tuple[str, str]] = [ " ci-PLACEHOLDER-${{ github.event.pull_request.number || github.ref }}${{", ), (r'^ platform: "linux/(?:arm64|amd64)"$', ' platform: "linux/PLACEHOLDER"'), - (r"^ runner-type: '\[\"ubuntu-22\.04(?:-arm)?\"\]'$", " runner-type: 'PLACEHOLDER'"), (r'^ PLATFORM: "linux/(?:arm64|amd64)"$', ' PLATFORM: "linux/PLACEHOLDER"'), ( r"^ # (?:ARM|AMD) copy\)\. The matching (?:AMD|ARM) copy lives in ci-(?:amd|arm)\.yml\.$",
![https://assets2.astronomer.io/logos/logoForLIGHTbackground.png"](https://assets2.astronomer.io/logos/logoForLIGHTbackground.png"){kind=link}
![https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/[email protected]?raw=true"](https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/aws-cloud-alt_light...@4x.png?raw=true"){kind=link}