This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch switch-cache-building-to-public-macos-runners
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit af971f08943451ef8150db8b851a50f9b73d3883
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 30 23:19:27 2024 +0100
Switch building images for ARM to public GitHub macos-14 runners
GitHub recently added a possibility for OSS projects to run the
macos-14 runners for ARM building. We are switching to build cache
images using those runners.
---
.github/workflows/additional-ci-image-checks.yml | 6 +++++-
.github/workflows/ci.yml | 5 ++++-
.github/workflows/finalize-tests.yml | 6 +++++-
.github/workflows/push-image-cache.yml | 6 ------
dev/breeze/src/airflow_breeze/global_constants.py | 1 +
dev/breeze/src/airflow_breeze/utils/selective_checks.py | 14 +++++++++++---
6 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 2331ceba9c..877de552ff 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy
description: "The array of labels (in json form) determining default
runner used for the build."
required: true
type: string
+ macos-runs-on-as-string:
+ description: "The array of labels (in json form) determining macos
runners."
+ required: true
+ type: string
public-runs-on-as-string:
description: "The array of labels (in json form) determining public
runners."
required: true
@@ -96,7 +100,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on-as-string: ${{ inputs.public-runs-on-as-string }}
+ runs-on-as-string: ${{ inputs.macos-runs-on-as-string }}
cache-type: "Early"
include-prod-images: "false"
push-latest-images: "false"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 361fbd4674..6436c17e92 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -103,8 +103,9 @@ jobs:
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
default-runs-on-as-string: ${{
steps.selective-checks.outputs.default-runs-on-as-string }}
- self-hosted-runs-on-as-string: ${{
steps.selective-checks.outputs.self-hosted-runs-on-as-string }}
+ macos-runs-on-as-string: ${{
steps.selective-checks.outputs.macos-runs-on-as-string }}
public-runs-on-as-string: ${{
steps.selective-checks.outputs.public-runs-on-as-string }}
+ self-hosted-runs-on-as-string: ${{
steps.selective-checks.outputs.self-hosted-runs-on-as-string }}
is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
@@ -239,6 +240,7 @@ jobs:
if: needs.build-info.outputs.canary-run == 'true'
with:
default-runs-on-as-string: ${{
needs.build-info.outputs.default-runs-on-as-string }}
+ macos-runs-on-as-string: ${{
needs.build-info.outputs.macos-runs-on-as-string }}
public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -609,6 +611,7 @@ jobs:
uses: ./.github/workflows/finalize-tests.yml
with:
public-runs-on-as-string: ${{
needs.build-info.outputs.public-runs-on-as-string }}
+ macos-runs-on-as-string: ${{
needs.build-info.outputs.macos-runs-on-as-string }}
self-hosted-runs-on-as-string: ${{
needs.build-info.outputs.self-hosted-runs-on-as-string }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index aa8e34bf48..c000a4f0b8 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy
description: "The array of labels (in json form) determining public
runners."
required: true
type: string
+ macos-runs-on-as-string:
+ description: "The array of labels (in json form) determining macos
runners."
+ required: true
+ type: string
self-hosted-runs-on-as-string:
description: "The array of labels (in json form) determining public
runners."
required: true
@@ -158,7 +162,7 @@ jobs:
packages: write
secrets: inherit
with:
- runs-on-as-string: ${{ inputs.self-hosted-runs-on-as-string }}
+ runs-on-as-string: ${{ inputs.macos-runs-on-as-string }}
cache-type: "Regular ARM"
include-prod-images: "true"
push-latest-images: "true"
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 94cd9eb9d1..3b0b5dce10 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -178,9 +178,6 @@ jobs:
with:
name: prod-packages
path: ./docker-context-files
- - name: "Start ARM instance"
- run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- name: "Push PROD ${{ inputs.cache-type }} cache: ${{
matrix.python-version }} ${{ inputs.platform }}"
@@ -189,9 +186,6 @@ jobs:
--prepare-buildx-cache --platform "${{ inputs.platform }}"
--install-packages-from-context --airflow-constraints-mode
constraints-source-providers
--python ${{ matrix.python }}
- - name: "Stop ARM instance"
- run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.platform == 'linux/arm64'
# We only push "AMD" images as it is really only needed for any kind
of automated builds in CI
# and currently there is not an easy way to make multi-platform image
from two separate builds
# and we can do it after we stopped the ARM instance as it is not
needed anymore
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 745913e038..71267bd47d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -30,6 +30,7 @@ from typing import Iterable
from airflow_breeze.utils.host_info_utils import Architecture
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+RUNS_ON_MACOS_RUNNER = '["macos-14"]'
RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
# we should get more sophisticated logic here in the future, but for now we
just check if
# we use self airflow, vm-based, amd hosted runner as a default
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 2b318d6f43..bf382b6c5f 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -43,6 +43,7 @@ from airflow_breeze.global_constants import (
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
HELM_VERSION,
KIND_VERSION,
+ RUNS_ON_MACOS_RUNNER,
RUNS_ON_PUBLIC_RUNNER,
RUNS_ON_SELF_HOSTED_RUNNER,
GithubEvents,
@@ -1133,6 +1134,10 @@ class SelectiveChecks:
def public_runs_on_as_string(self) -> str:
return RUNS_ON_PUBLIC_RUNNER
+ @cached_property
+ def macos_runs_on_as_string(self) -> str:
+ return RUNS_ON_MACOS_RUNNER
+
@cached_property
def is_self_hosted_runner(self) -> bool:
"""
@@ -1172,7 +1177,7 @@ class SelectiveChecks:
or "x64" == label.lower()
or "asf-runner" == label
or ("ubuntu" in label and "arm" not in label.lower())
- for label in json.loads(self.public_runs_on_as_string)
+ for label in json.loads(self.default_runs_on_as_string)
]
)
@@ -1187,8 +1192,11 @@ class SelectiveChecks:
"""
return any(
[
- "arm" == label.lower() or "arm64" == label.lower() or
"asf-arm" == label
- for label in json.loads(self.public_runs_on_as_string)
+ "arm" == label.lower()
+ or "arm64" == label.lower()
+ or "asf-arm" == label
+ or "macos-14" == label
+ for label in json.loads(self.default_runs_on_as_string)
]
)
