This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch consistent-approach-for-runs-on-usage
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6ad6c1b4a276caae4bed4e56c4381cd2aaebfdf7
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu Mar 28 12:58:46 2024 +0100
Use explicit and easier to use runs-on approach for CI workflows
Depending on selective checks, but also on the job executed, we
choose whether to run job on public runners or self-hosted runners.
So far the set of labels to select the runners were passed in a bit
inconsistent way. Outputs of selective checks can only be strings
and the `run-as` accepts array of strings (labels) - so we were
using fromJSON to convert between the two. And we used runs-on
inputs on a number of our workflows to pass the selection.
However this meant that runs-on could be either string or array and
that sometimes we passed public/self-hosted labels as strings
directly and some of those were hard-coded.
This PR changes it consistently across the board to introduce
consistent approach:
* build info have no selective checks results yet, so for them
runs-on is hardcoded
* similarly for "windows" and release jobs that are manually run
without running selective checks
* selective checks will produce three outuputs - JSON stringiified
array of labels:
* default (one that is selected depending on who runs the build)
* public (for cases where we want to force the builds to use public
runners
* self-hosted (for cases where we want to force the builds to use
self-hosted runners
* all the outputs are named `<type>-runs-on-as-string` to make it clear
they are all strings
* all inputs of workflows expectings strings are named the same (with
as-string suffix and <type> prefix)
* whenever a job is run, we pass "runs-on" parameter to be `fromJSON`
with appropriate type we want to use passed as input
This will make it easier to reason on which job is using which type
of runner and it will make it easier in the future to make it more
flexible when we add ASF self-hosted runners and possibly our own
K8S runners, or when we would want to change labels for public runners
or self-hosted runners.
---
.github/workflows/additional-ci-image-checks.yml | 17 ++++----
.github/workflows/additional-prod-image-tests.yml | 13 +++---
.github/workflows/basic-tests.yml | 16 +++++---
.github/workflows/build-images.yml | 10 +++--
.github/workflows/check-providers.yml | 13 +++---
.github/workflows/ci-image-build.yml | 9 ++--
.github/workflows/ci.yml | 48 ++++++++++++----------
.github/workflows/finalize-tests.yml | 19 +++++----
.github/workflows/generate-constraints.yml | 6 ++-
.github/workflows/helm-tests.yml | 15 ++++---
.github/workflows/integration-tests.yml | 9 ++--
.github/workflows/k8s-tests.yml | 9 ++--
.github/workflows/prod-image-build.yml | 11 +++--
.github/workflows/prod-image-extra-checks.yml | 7 ++++
.github/workflows/push-image-cache.yml | 11 +++--
.github/workflows/release_dockerhub_image.yml | 4 --
.github/workflows/run-unit-tests.yml | 9 ++--
.github/workflows/special-tests.yml | 23 +++++------
.github/workflows/static-checks-mypy-docs.yml | 13 +++---
dev/breeze/doc/ci/04_selective_checks.md | 6 ++-
.../src/airflow_breeze/utils/selective_checks.py | 18 +++++---
dev/breeze/tests/test_selective_checks.py | 14 +++----
22 files changed, 164 insertions(+), 136 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 8ff00ba617..2331ceba9c 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -20,10 +20,13 @@ name: Additional CI image checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
+ type: string
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -93,7 +96,7 @@ jobs:
packages: write
secrets: inherit
with:
- # Runs on Public runners
+ runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
cache-type: "Early"
include-prod-images: "false"
push-latest-images: "false"
@@ -110,7 +113,7 @@ jobs:
check-that-image-builds-quickly:
timeout-minutes: 11
name: Check that image builds quickly
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
@@ -150,7 +153,7 @@ jobs:
secrets: inherit
with:
push-image: "false"
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
image-tag: ${{ inputs.image-tag }}
python-versions: ${{ inputs.python-versions }}
platform: "linux/arm64"
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index c6a2146ecf..548a897c1b 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -20,10 +20,9 @@ name: Additional PROD image tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
type: string
default-branch:
description: "The default branch for the repository"
@@ -62,6 +61,7 @@ jobs:
name: PROD image extra checks (main)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
+ public-runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
@@ -77,6 +77,7 @@ jobs:
name: PROD image extra checks (release)
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
+ public-runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
python-versions: "[ '${{ inputs.default-python-version }}' ]"
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
@@ -91,7 +92,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of POD image building"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -135,7 +136,7 @@ jobs:
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker-compose quick start with PROD image verifying"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
env:
IMAGE_TAG: "${{ inputs.image-tag }}"
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index f96997c957..37cd8e7d3c 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -20,6 +20,10 @@ name: Basic tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
+ type: string
run-www-tests:
description: "Whether to run WWW tests (true/false)"
required: true
@@ -52,7 +56,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -76,7 +80,7 @@ jobs:
tests-www:
timeout-minutes: 10
name: React WWW tests
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
if: inputs.run-www-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -105,7 +109,7 @@ jobs:
test-openapi-client:
timeout-minutes: 10
name: "Test OpenAPI client"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
if: inputs.needs-api-codegen == 'true'
steps:
- name: "Cleanup repo"
@@ -195,7 +199,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -262,7 +266,7 @@ jobs:
upgrade-check:
timeout-minutes: 45
name: "Upgrade checks"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -330,7 +334,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(inputs.public-runs-on-as-string) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
IMAGE_TAG: ${{ inputs.image-tag }}
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index b135da93cb..de51cdecc7 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -51,6 +51,7 @@ jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
+ # At build-info stage we do not yet have outputs so we need to hard-code
the runs-on to public runners
runs-on: ["ubuntu-22.04"]
env:
TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
@@ -67,7 +68,9 @@ jobs:
docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
- runs-on: ${{steps.selective-checks.outputs.runs-on}}
+ default-runs-on-as-string: ${{
steps.selective-checks.outputs.default-runs-on-as-string }}
+ self-hosted-runs-on-as-string: ${{
steps.selective-checks.outputs.self-hosted-runs-on-as-string }}
+ public-runs-on-as-string: ${{
steps.selective-checks.outputs.public-runs-on-as-string }}
is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
is-committer-build: ${{
steps.selective-checks.outputs.is-committer-build }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
@@ -171,7 +174,7 @@ jobs:
needs.build-info.outputs.ci-image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
with:
- runs-on: '["ubuntu-22.04"]'
+ default-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
do-build: ${{ needs.build-info.outputs.ci-image-build }}
target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }}
pull-request-target: "true"
@@ -191,6 +194,7 @@ jobs:
needs: [build-info, build-ci-images]
uses: ./.github/workflows/generate-constraints.yml
with:
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
# For regular PRs we do not need "no providers" constraints - they are
only needed in canary builds
generate-no-providers-constraints: "false"
@@ -211,7 +215,7 @@ jobs:
needs.build-info.outputs.prod-image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
with:
- runs-on: '["ubuntu-22.04"]'
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.ci-image-build }}
upload-package-artifact: "true"
diff --git a/.github/workflows/check-providers.yml
b/.github/workflows/check-providers.yml
index 7c62d5ab52..72ceb7106a 100644
--- a/.github/workflows/check-providers.yml
+++ b/.github/workflows/check-providers.yml
@@ -20,10 +20,9 @@ name: Provider tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -59,7 +58,7 @@ jobs:
prepare-install-verify-provider-packages-wheel:
timeout-minutes: 80
name: "Provider packages wheel build and verify"
- runs-on: ${{fromJSON(inputs.runs-on)}}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -124,7 +123,7 @@ jobs:
prepare-install-provider-packages-sdist:
timeout-minutes: 80
name: "Provider packages sdist build and install"
- runs-on: ${{fromJSON(inputs.runs-on)}}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -192,7 +191,7 @@ jobs:
providers-compatibility-checks:
timeout-minutes: 80
name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }}
provider check
- runs-on: ${{fromJSON(inputs.runs-on)}}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 75a611ba55..d7d09d2a2b 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -20,10 +20,9 @@ name: Build CI images
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
do-build:
description: >
@@ -105,7 +104,7 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building'
}} \
CI ${{ inputs.platform }} image\
${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
env:
BACKEND: sqlite
DEFAULT_BRANCH: ${{ inputs.branch }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 438422aaf6..bb89c6bb23 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,10 +48,7 @@ jobs:
build-info:
name: "Build info"
- # The runs-on cannot refer to env. or secrets. context, so we have no
- # option but to specify a hard-coded list here. This is "safe", the
- # runner checks if the user is an owner or collaborator of the repo
- # before running the workflow.
+ # At build-info stage we do not yet have outputs so we need to hard-code
the runs-on to public runners
runs-on: ["ubuntu-22.04"]
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
@@ -105,7 +102,9 @@ jobs:
providers-compatibility-checks: ${{
steps.selective-checks.outputs.providers-compatibility-checks }}
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
- runs-on: ${{steps.selective-checks.outputs.runs-on}}
+ default-runs-on-as-string: ${{
steps.selective-checks.outputs.default-runs-on-as-string }}
+ self-hosted-runs-on-as-string: ${{
steps.selective-checks.outputs.self-hosted-runs-on-as-string }}
+ public-runs-on-as-string: ${{
steps.selective-checks.outputs.public-runs-on-as-string }}
is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
@@ -160,6 +159,7 @@ jobs:
needs: [build-info]
uses: ./.github/workflows/basic-tests.yml
with:
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
default-python-version:
${{needs.build-info.outputs.default-python-version}}
@@ -183,7 +183,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on: '["ubuntu-22.04"]'
+ default-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
platform: "linux/amd64"
@@ -197,7 +197,7 @@ jobs:
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJSON(needs.build-info.outputs.default-runs-on-as-string)
}}
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.ci-image-build == 'true'
env:
@@ -238,7 +238,8 @@ jobs:
uses: ./.github/workflows/additional-ci-image-checks.yml
if: needs.build-info.outputs.canary-run == 'true'
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
@@ -259,6 +260,7 @@ jobs:
uses: ./.github/workflows/generate-constraints.yml
if: needs.build-info.outputs.ci-image-build == 'true'
with:
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
# generate no providers constraints only in canary builds - they take
quite some time to generate
# they are not needed for regular builds, they are only needed to update
constraints in canaries
@@ -273,7 +275,7 @@ jobs:
uses: ./.github/workflows/static-checks-mypy-docs.yml
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
@@ -303,7 +305,7 @@ jobs:
needs.build-info.outputs.skip-providers-tests != 'true' &&
needs.build-info.outputs.latest-versions-only != 'true'
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
@@ -321,7 +323,8 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
@@ -339,7 +342,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
@@ -363,7 +366,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
@@ -387,7 +390,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
@@ -413,7 +416,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
@@ -443,7 +446,7 @@ jobs:
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' ||
needs.build-info.outputs.full-tests-needed == 'true')
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
@@ -462,7 +465,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
@@ -489,7 +492,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on: '["ubuntu-22.04"]'
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
upload-package-artifact: "true"
@@ -509,7 +512,7 @@ jobs:
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJson(needs.build-info.outputs.public-runs-on-as-string) }}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
env:
@@ -554,7 +557,7 @@ jobs:
needs: [build-info, wait-for-prod-images, generate-constraints]
uses: ./.github/workflows/additional-prod-image-tests.yml
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
@@ -574,7 +577,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
kubernetes-versions-list-as-string: ${{
needs.build-info.outputs.kubernetes-versions-list-as-string }}
@@ -605,7 +608,8 @@ jobs:
- tests-special
uses: ./.github/workflows/finalize-tests.yml
with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
+ public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
+ self-hosted-runs-on-as-string: ${{
needs.build-info.outputs.self-hosted-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index 504f4a7698..aa8e34bf48 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -20,10 +20,13 @@ name: Finalize tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
+ type: string
+ self-hosted-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -71,7 +74,7 @@ on: # yamllint disable-line rule:truthy
type: string
jobs:
update-constraints:
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
timeout-minutes: 80
name: "Update constraints"
permissions:
@@ -133,7 +136,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on: '["ubuntu-22.04"]'
+ runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
cache-type: "Regular AMD"
include-prod-images: "true"
push-latest-images: "true"
@@ -155,7 +158,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ runs-on-as-string: ${{ inputs.self-hosted-runs-on-as-string }}
cache-type: "Regular ARM"
include-prod-images: "true"
push-latest-images: "true"
@@ -171,7 +174,7 @@ jobs:
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
steps:
- name: "Cleanup repo"
shell: bash
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 0adad4da3b..30d4a21c10 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -20,6 +20,10 @@ name: Generate constraints
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
+ type: string
python-versions-list-as-string:
description: "Stringified array of all Python versions to test -
separated by spaces."
required: true
@@ -46,7 +50,7 @@ jobs:
contents: read
timeout-minutes: 70
name: Generate constraints ${{ inputs.python-versions-list-as-string }}
- runs-on: ['ubuntu-22.04']
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
env:
DEBUG_RESOURCES: ${{ inputs.debug-resources }}
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index f6a2df6041..ffa7b05f6b 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -20,10 +20,13 @@ name: Helm tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
+ type: string
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
type: string
helm-test-packages:
description: "Stringified JSON array of helm test packages to test"
@@ -41,7 +44,7 @@ jobs:
tests-helm:
timeout-minutes: 80
name: "Unit tests Helm: ${{ matrix.helm-test-package }}"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
@@ -77,7 +80,7 @@ jobs:
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
steps:
diff --git a/.github/workflows/integration-tests.yml
b/.github/workflows/integration-tests.yml
index abc3c631f1..eadf53eb40 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -20,10 +20,9 @@ name: Integration tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -65,7 +64,7 @@ jobs:
tests-integration:
timeout-minutes: 130
name: Integration Tests ${{ matrix.backend }}:${{ matrix.backend-version}}
- runs-on: ${{fromJSON(inputs.runs-on)}}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
index 7e09efa994..9a5574f0f3 100644
--- a/.github/workflows/k8s-tests.yml
+++ b/.github/workflows/k8s-tests.yml
@@ -20,10 +20,9 @@ name: K8s tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -55,7 +54,7 @@ jobs:
name: "\
K8S System:${{ matrix.executor }} - ${{ matrix.use-standard-naming }} - \
${{ inputs.kubernetes-versions-list-as-string }}"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index ad9c35a445..602111cdaf 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -20,10 +20,9 @@ name: Build PROD images
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
build-type:
description: >
@@ -120,7 +119,7 @@ jobs:
build-prod-packages:
name: "${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }}
Airflow and provider packages"
timeout-minutes: 10
- runs-on: ["ubuntu-22.04"]
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
steps:
@@ -196,7 +195,7 @@ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building'
}} \
PROD ${{ inputs.build-type }} image\
${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.public-runs-on-as-string) }}
needs:
- build-prod-packages
env:
diff --git a/.github/workflows/prod-image-extra-checks.yml
b/.github/workflows/prod-image-extra-checks.yml
index 48469af793..153837ac1c 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -20,6 +20,10 @@ name: PROD images extra checks
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
+ public-runs-on-as-string:
+ description: "The array of labels (in json form) determining public
runners."
+ required: true
+ type: string
python-versions:
description: "JSON-formatted array of Python versions to build images
from"
required: true
@@ -59,6 +63,7 @@ jobs:
bullseye-image:
uses: ./.github/workflows/prod-image-build.yml
with:
+ public-runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
build-type: "Bullseye"
upload-package-artifact: "false"
image-tag: bullseye-${{ inputs.image-tag }}
@@ -79,6 +84,7 @@ jobs:
myssql-client-image:
uses: ./.github/workflows/prod-image-build.yml
with:
+ public-runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
build-type: "MySQL Client"
upload-package-artifact: "false"
image-tag: mysql-${{ inputs.image-tag }}
@@ -99,6 +105,7 @@ jobs:
pip-image:
uses: ./.github/workflows/prod-image-build.yml
with:
+ public-runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
build-type: "pip"
upload-package-artifact: "false"
image-tag: mysql-${{ inputs.image-tag }}
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index cb8c072e74..94cd9eb9d1 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -20,10 +20,9 @@ name: Push image cache
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: "[\"ubuntu-22.04\"]"
+ runs-on-as-string:
+ description: "The array of labels (in json form) determining where the
job should run on."
+ required: true
type: string
cache-type:
description: "Type of cache to push (Early / Regular)."
@@ -76,7 +75,7 @@ on: # yamllint disable-line rule:truthy
jobs:
push-ci-image-cache:
name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
@@ -134,7 +133,7 @@ jobs:
push-prod-image-cache:
name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 2ce069cc0c..5eb9c6ca49 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -40,8 +40,6 @@ jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
- # TODO: when we have it properly set-up with labels we should check for
- # "airflow-runner" presence in runs_on
runs-on: ["ubuntu-22.04"]
outputs:
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
@@ -73,8 +71,6 @@ jobs:
release-images:
timeout-minutes: 120
name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}"
- # TODO: when we have it properly set-up with labels we should check for
- # "airflow-runner" presence in runs_on
runs-on: ["self-hosted", "Linux", "X64"]
needs: [build-info]
strategy:
diff --git a/.github/workflows/run-unit-tests.yml
b/.github/workflows/run-unit-tests.yml
index a54019da21..3a650610df 100644
--- a/.github/workflows/run-unit-tests.yml
+++ b/.github/workflows/run-unit-tests.yml
@@ -20,10 +20,9 @@ name: Unit tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
backend:
description: "The backend to run the tests on"
@@ -112,7 +111,7 @@ jobs:
${{ inputs.test-scope }}:\
${{ inputs.test-name }}${{ inputs.test-name-separator }}${{
matrix.backend-version }}:\
${{matrix.python-version}}: ${{
inputs.parallel-test-types-list-as-string }}"
- runs-on: ${{fromJSON(inputs.runs-on)}}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index a644c6d661..c35fdc5a7b 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -20,10 +20,9 @@ name: Special tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -66,7 +65,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
test-scope: "DB"
@@ -88,7 +87,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
test-scope: "All"
@@ -110,7 +109,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
pydantic: "v1"
test-name: "Pydantic-V1-Postgres"
test-scope: "All"
@@ -132,7 +131,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
pydantic: "none"
test-name: "Pydantic-Removed-Postgres"
test-scope: "All"
@@ -154,7 +153,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
test-scope: "All"
@@ -176,7 +175,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
enable-aip-44: "false"
test-name: "InProgressDisabled-Postgres"
test-scope: "All"
@@ -198,7 +197,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
test-name: "Postgres"
test-scope: "Quarantined"
backend: "postgres"
@@ -219,7 +218,7 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on: ${{ inputs.runs-on }}
+ default-runs-on-as-string: ${{ inputs.default-runs-on-as-string }}
test-name: "Postgres"
test-scope: "ARM collection"
backend: "postgres"
diff --git a/.github/workflows/static-checks-mypy-docs.yml
b/.github/workflows/static-checks-mypy-docs.yml
index 2dcc64c9c3..cb7f53ab22 100644
--- a/.github/workflows/static-checks-mypy-docs.yml
+++ b/.github/workflows/static-checks-mypy-docs.yml
@@ -20,10 +20,9 @@ name: Static checks, mypy, docs
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on:
- description: "The array of labels (in json form) determining type of
the runner to use for the build."
- required: false
- default: '["ubuntu-22.04"]'
+ default-runs-on-as-string:
+ description: "The array of labels (in json form) determining default
runner used for the build."
+ required: true
type: string
image-tag:
description: "Tag to set for the image"
@@ -93,7 +92,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies
}}"
@@ -140,7 +139,7 @@ jobs:
mypy:
timeout-minutes: 45
name: "MyPy checks"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
if: inputs.needs-mypy == 'true'
strategy:
fail-fast: false
@@ -178,7 +177,7 @@ jobs:
build-docs:
timeout-minutes: 60
name: "Build documentation"
- runs-on: ${{ fromJSON(inputs.runs-on) }}
+ runs-on: ${{ fromJSON(inputs.default-runs-on-as-string) }}
strategy:
fail-fast: false
matrix:
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index b558061208..a1c17b41ce 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -223,7 +223,9 @@ Github Actions to pass the list of parameters to a command
to execute
| run-kubernetes-tests | Whether Kubernetes tests should be
run ("true"/"false") | true
| |
| run-tests | Whether unit tests should be run
("true"/"false") | true
| |
| run-www-tests | Whether WWW tests should be run
("true"/"false") | true
| |
-| runs-on | List of labels assigned for runners
for that build (used to select runners) |
["ubuntu-22.04"] | |
+| default-runs-on-as-string | List of labels assigned for runners
for that build for default runs for that build (as string) |
["ubuntu-22.04"] | |
+| self-hosted-runs-on-as-string | List of labels assigned for runners
for that build for self hosted runners |
["self-hosted", "Linux", "X64"] | |
+| public-runs-on-as-string | List of labels assigned for runners
for that build for public runners |
["ubuntu-22.04"] | |
| skip-pre-commits | Which pre-commits should be skipped
during the static-checks run |
check-provider-yaml-valid,flynt,identity | |
| skip-provider-tests | When provider tests should be
skipped (on non-main branch or when no provider changes detected) | true
| |
| sqlite-exclude | Which versions of Sqlite to exclude
for tests as JSON array | []
| |
@@ -308,7 +310,7 @@ am overview of possible labels and their meaning:
| full tests needed | full-tests-needed | Run complete
set of tests (might be with default or all python/k8s versions)
|
| non committer build | is-committer-build | If set then
even for non-committer builds, the scripts used for images are used from target
branch. |
| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set then
dependencies in the CI image build are upgraded to the newer ones.
|
-| use public runners | runs-on | Force using
public runners even for Committer runs.
|
+| use public runners | default-runs-on-as-string | Force using
public runners even for Committer runs.
|
-----
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index d38cf61e4d..2b318d6f43 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1103,7 +1103,7 @@ class SelectiveChecks:
return " ".join(sorted(affected_providers))
@cached_property
- def runs_on(self) -> str:
+ def default_runs_on_as_string(self) -> str:
if self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY:
if self._github_event in [GithubEvents.SCHEDULE,
GithubEvents.PUSH]:
return RUNS_ON_SELF_HOSTED_RUNNER
@@ -1125,6 +1125,14 @@ class SelectiveChecks:
return RUNS_ON_SELF_HOSTED_RUNNER
return RUNS_ON_PUBLIC_RUNNER
+ @cached_property
+ def self_hosted_runs_on_as_string(self) -> str:
+ return RUNS_ON_SELF_HOSTED_RUNNER
+
+ @cached_property
+ def public_runs_on_as_string(self) -> str:
+ return RUNS_ON_PUBLIC_RUNNER
+
@cached_property
def is_self_hosted_runner(self) -> bool:
"""
@@ -1132,7 +1140,7 @@ class SelectiveChecks:
All self-hosted runners have "self-hosted" label.
"""
- return "self-hosted" in json.loads(self.runs_on)
+ return "self-hosted" in json.loads(self.default_runs_on_as_string)
@cached_property
def is_airflow_runner(self) -> bool:
@@ -1143,7 +1151,7 @@ class SelectiveChecks:
"""
# TODO: when we have it properly set-up with labels we should just
check for
# "airflow-runner" presence in runs_on
- runs_on_array = json.loads(self.runs_on)
+ runs_on_array = json.loads(self.default_runs_on_as_string)
return "Linux" in runs_on_array and "X64" in runs_on_array and
"self-hosted" in runs_on_array
@cached_property
@@ -1164,7 +1172,7 @@ class SelectiveChecks:
or "x64" == label.lower()
or "asf-runner" == label
or ("ubuntu" in label and "arm" not in label.lower())
- for label in json.loads(self.runs_on)
+ for label in json.loads(self.public_runs_on_as_string)
]
)
@@ -1180,7 +1188,7 @@ class SelectiveChecks:
return any(
[
"arm" == label.lower() or "arm64" == label.lower() or
"asf-arm" == label
- for label in json.loads(self.runs_on)
+ for label in json.loads(self.public_runs_on_as_string)
]
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 441abdb1be..c29fc9a440 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1624,9 +1624,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
@pytest.mark.parametrize(
- "github_event, github_actor, github_repository, pr_labels,
github_context_dict, "
- "runs_on, "
- "is_self_hosted_runner, is_airflow_runner, is_amd_runner, is_arm_runner,
is_vm_runner, is_k8s_runner",
+ "github_event, github_actor, github_repository, pr_labels,
github_context_dict, default_runs_on_as_string, is_self_hosted_runner,
is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner",
[
pytest.param(
GithubEvents.PUSH,
@@ -1816,7 +1814,7 @@ def test_runs_on(
github_repository: str,
pr_labels: list[str],
github_context_dict: dict[str, Any],
- runs_on: str,
+ default_runs_on_as_string,
is_self_hosted_runner: str,
is_airflow_runner: str,
is_amd_runner: str,
@@ -1834,7 +1832,7 @@ def test_runs_on(
pr_labels=(),
default_branch="main",
)
- assert_outputs_are_printed({"runs-on": runs_on}, str(stderr))
+ assert_outputs_are_printed({"default-runs-on-as-string":
default_runs_on_as_string}, str(stderr))
assert_outputs_are_printed({"is-self-hosted-runner":
is_self_hosted_runner}, str(stderr))
assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner},
str(stderr))
assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr))
@@ -1999,7 +1997,7 @@ def test_mypy_matches(
("README.md",),
{
"is-committer-build": "false",
- "runs-on": '["ubuntu-22.04"]',
+ "default-runs-on-as-string": '["ubuntu-22.04"]',
},
"",
(),
@@ -2009,7 +2007,7 @@ def test_mypy_matches(
("README.md",),
{
"is-committer-build": "true",
- "runs-on": '["self-hosted", "Linux", "X64"]',
+ "default-runs-on-as-string": '["self-hosted", "Linux", "X64"]',
},
"potiuk",
(),
@@ -2019,7 +2017,7 @@ def test_mypy_matches(
("README.md",),
{
"is-committer-build": "false",
- "runs-on": '["self-hosted", "Linux", "X64"]',
+ "default-runs-on-as-string": '["self-hosted", "Linux", "X64"]',
},
"potiuk",
("non committer build",),
