This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57a051997a0 Fix linux/arm64 CI image build broken by the ibm.db2
provider (#72058)
57a051997a0 is described below
commit 57a051997a01300b48cc6b180040418fb13b435e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Aug 26 13:12:23 2026 +0200
Fix linux/arm64 CI image build broken by the ibm.db2 provider (#72058)
`ibm-db` publishes no Linux aarch64 wheel and its sdist fails to build there
with `NameError: name 'arch_' is not defined`, so every linux/arm64 CI image
build has failed since the provider merged, blocking the scheduled ARM run
on
main.
Apply the mechanism already used for ibm.mq — declare `excluded-platforms`
so
the provider leaves the ARM test matrix, and translate that into
install-time
markers so it is never pulled in there. `ibm-db-sa` is pure Python but
depends
on `ibm-db`, so it needs the same marker.
That mechanism mapped `linux/arm64` to both `aarch64` and `arm64`, which is
wrong in general: `platform.machine()` reports `aarch64` on Linux and
`arm64`
on macOS, so the two are distinct targets rather than two spellings of one.
IBM does ship macosx arm64 wheels for `ibm-db`, so excluding `arm64` would
have
removed a working driver from Apple Silicon. Each platform now contributes
only
its own machine spelling, and a provider unusable on every ARM target says
so by
listing both platforms — which is what ibm.mq now does, leaving its
generated
markers unchanged.
---
airflow-core/src/airflow/provider.yaml.schema.json | 2 +-
dev/breeze/doc/ci/04_selective_checks.md | 7 +++++--
dev/breeze/tests/test_selective_checks.py | 8 +++++---
providers/ibm/db2/README.rst | 6 +++++-
providers/ibm/db2/docs/index.rst | 10 +++++-----
providers/ibm/db2/provider.yaml | 6 ++++++
providers/ibm/db2/pyproject.toml | 8 ++++++--
providers/ibm/mq/provider.yaml | 7 ++++---
scripts/ci/prek/common_prek_utils.py | 18 ++++++++++++------
.../ci/prek/test_check_excluded_provider_markers.py | 19 ++++++++++++++++++-
uv.lock | 8 ++++----
11 files changed, 71 insertions(+), 28 deletions(-)
diff --git a/airflow-core/src/airflow/provider.yaml.schema.json
b/airflow-core/src/airflow/provider.yaml.schema.json
index a8a08c4d71e..fd449394c18 100644
--- a/airflow-core/src/airflow/provider.yaml.schema.json
+++ b/airflow-core/src/airflow/provider.yaml.schema.json
@@ -59,7 +59,7 @@
}
},
"excluded-platforms": {
- "description": "List of platforms (e.g. linux/arm64) excluded for
that provider. Used to skip providers whose native dependencies are unavailable
on a given architecture.",
+ "description": "List of platforms (e.g. linux/arm64, darwin/arm64)
excluded for that provider. Used to skip providers whose native dependencies
are unavailable on a given architecture: linux/* values also drop the provider
from that CI test matrix, while every value contributes a platform_machine
install marker.",
"type": "array",
"items": {
"type": "string"
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index be4fa4d4c82..27e6039996f 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -233,8 +233,11 @@ Some integrations and providers only work on one CPU
architecture, so the select
by the platform the run's tests will execute on:
* **Integrations** in `DISABLE_TESTABLE_INTEGRATIONS_FROM_ARM` are dropped on
ARM.
-* **Providers** that declare `excluded-platforms` in their `provider.yaml`
(e.g. `ibm.mq` excludes
- `linux/arm64`) are removed from the providers test-type matrix on that
platform.
+* **Providers** that declare `excluded-platforms` in their `provider.yaml`
(e.g. `ibm.mq` and
+ `ibm.db2` exclude `linux/arm64`) are removed from the providers test-type
matrix on that platform.
+ Only `linux/*` values can match a run; a provider may also list a non-CI
platform such as
+ `darwin/arm64`, which never affects the matrix and exists solely to add an
install-time
+ `platform_machine` marker.
## Individually simple rules
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index a4cf50bd8d8..0e45e016b48 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -3635,9 +3635,9 @@ def
test_testable_providers_integrations_gated_by_affected_provider():
def test_individual_providers_excludes_platform_excluded_on_arm():
- """ibm.mq declares `excluded-platforms: [linux/arm64]`, so it must be
absent from
- the ARM individual-providers matrix (used by the Low-dep ARM canary job)
and
- present on AMD."""
+ """ibm.mq and ibm.db2 declare `excluded-platforms: [linux/arm64]`, so they
must be
+ absent from the ARM individual-providers matrix (used by the Low-dep ARM
canary job)
+ and present on AMD."""
arm_checks = SelectiveChecks(
files=("airflow-core/tests/test_example.py",),
commit_ref=NEUTRAL_COMMIT,
@@ -3650,6 +3650,7 @@ def
test_individual_providers_excludes_platform_excluded_on_arm():
arm_output =
arm_checks.individual_providers_test_types_list_as_strings_in_json
assert arm_output is not None
assert "Providers[ibm.mq]" not in arm_output
+ assert "Providers[ibm.db2]" not in arm_output
amd_checks = SelectiveChecks(
files=("airflow-core/tests/test_example.py",),
@@ -3663,6 +3664,7 @@ def
test_individual_providers_excludes_platform_excluded_on_arm():
amd_output =
amd_checks.individual_providers_test_types_list_as_strings_in_json
assert amd_output is not None
assert "Providers[ibm.mq]" in amd_output
+ assert "Providers[ibm.db2]" in amd_output
def test_run_kubernetes_tests_forced_by_label():
diff --git a/providers/ibm/db2/README.rst b/providers/ibm/db2/README.rst
index 789ce2016d8..679d2d21ecb 100644
--- a/providers/ibm/db2/README.rst
+++ b/providers/ibm/db2/README.rst
@@ -34,9 +34,13 @@ Requirements
This provider requires the following packages:
-- ``ibm-db>=3.0.0`` - IBM Db2 Python driver
+- ``ibm-db>=3.2.0`` - IBM Db2 Python driver
- ``ibm-db-sa>=0.4.0`` - SQLAlchemy dialect for Db2
+IBM publishes no Db2 client for Linux ARM, so neither package is installed on
+``aarch64``. The provider itself is pure Python and still installs there, but
opening a
+Db2 connection fails with ``ModuleNotFoundError``. macOS Apple Silicon is
unaffected.
+
Configuration
-------------
diff --git a/providers/ibm/db2/docs/index.rst b/providers/ibm/db2/docs/index.rst
index b194c4ca611..86f4ba086a1 100644
--- a/providers/ibm/db2/docs/index.rst
+++ b/providers/ibm/db2/docs/index.rst
@@ -96,16 +96,16 @@ Requirements
The minimum Apache Airflow version supported by this provider distribution is
``2.11.0``.
-========================================== ==================
+==========================================
==========================================
PIP package Version required
-========================================== ==================
+==========================================
==========================================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``apache-airflow-providers-common-sql`` ``>=1.32.0``
-``ibm-db`` ``>=3.2.0``
-``ibm-db-sa`` ``>=0.4.0``
+``ibm-db`` ``>=3.2.0; platform_machine !=
"aarch64"``
+``ibm-db-sa`` ``>=0.4.0; platform_machine !=
"aarch64"``
``methodtools`` ``>=0.4.7``
-========================================== ==================
+==========================================
==========================================
Optional dependencies
---------------------
diff --git a/providers/ibm/db2/provider.yaml b/providers/ibm/db2/provider.yaml
index 3815e2c81c2..7f3ae51fe02 100644
--- a/providers/ibm/db2/provider.yaml
+++ b/providers/ibm/db2/provider.yaml
@@ -24,6 +24,12 @@ description: |
state: not-ready
lifecycle: incubation
source-date-epoch: 1743033600
+# IBM ships no Db2 client for Linux ARM64, so `ibm-db` publishes no aarch64
wheel and its
+# sdist build fails there with `NameError: name 'arch_' is not defined`,
breaking the
+# linux/arm64 CI image. Exclude the provider from the linux/arm64 test matrix
until IBM
+# publishes an aarch64 build.
+excluded-platforms:
+ - linux/arm64
versions:
- 0.1.0
diff --git a/providers/ibm/db2/pyproject.toml b/providers/ibm/db2/pyproject.toml
index 8153b93fd3e..3df1c6bac81 100644
--- a/providers/ibm/db2/pyproject.toml
+++ b/providers/ibm/db2/pyproject.toml
@@ -62,8 +62,12 @@ dependencies = [
"apache-airflow>=2.11.0",
"apache-airflow-providers-common-compat>=1.12.0",
"apache-airflow-providers-common-sql>=1.32.0",
- "ibm-db>=3.2.0",
- "ibm-db-sa>=0.4.0",
+ # `ibm-db` publishes no Linux aarch64 wheel and its sdist cannot be built
there, so
+ # skip it on Linux ARM (mirrors provider.yaml excluded-platforms). macOS
Apple Silicon
+ # is unaffected — IBM does ship macosx arm64 wheels. `ibm-db-sa` is pure
Python but
+ # depends on `ibm-db`, so it carries the same marker or it drags `ibm-db`
back in.
+ "ibm-db>=3.2.0; platform_machine != \"aarch64\"",
+ "ibm-db-sa>=0.4.0; platform_machine != \"aarch64\"",
"methodtools>=0.4.7",
]
diff --git a/providers/ibm/mq/provider.yaml b/providers/ibm/mq/provider.yaml
index 341980678aa..d18c5fa8e3b 100644
--- a/providers/ibm/mq/provider.yaml
+++ b/providers/ibm/mq/provider.yaml
@@ -27,11 +27,12 @@ description: |
# IBM MQ ships its native C client library only for Linux x86_64, Windows x64
and Java
# (https://www.ibm.com/support/pages/downloading-ibm-mq-94 — Redist clients).
The
# `ibmmq` Python bindings link against that C client at build time, so `uv sync
-# --resolution lowest-direct --all-extras` fails to build the sdist on aarch64.
-# Exclude the provider from the linux/arm64 test matrix until IBM publishes an
aarch64
-# redistributable.
+# --resolution lowest-direct --all-extras` fails to build the sdist on any ARM
machine.
+# Exclude the provider from the linux/arm64 test matrix, and from installs on
both ARM
+# targets, until IBM publishes an ARM redistributable.
excluded-platforms:
- linux/arm64
+ - darwin/arm64
# Note that those versions are maintained by release manager - do not update
them manually
# with the exception of case where other provider in sources has >= new
provider version.
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider
have
diff --git a/scripts/ci/prek/common_prek_utils.py
b/scripts/ci/prek/common_prek_utils.py
index 62154dedad0..40cd8ce5838 100644
--- a/scripts/ci/prek/common_prek_utils.py
+++ b/scripts/ci/prek/common_prek_utils.py
@@ -43,13 +43,19 @@ AIRFLOW_TASK_SDK_SOURCES_PATH = AIRFLOW_TASK_SDK_ROOT_PATH
/ "src"
DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10"
-# Maps a Docker build platform string (as declared in ``provider.yaml`` under
-# ``excluded-platforms``) to the ``platform_machine`` values Python reports on
that
-# architecture. ``linux/arm64`` covers both Linux (``aarch64``) and macOS
Apple Silicon
-# (``arm64``) so a provider opting out of ARM is never pulled in on any ARM
machine where
-# its native dependency cannot be built.
+# Maps a platform string (as declared in ``provider.yaml`` under
``excluded-platforms``)
+# to the ``platform_machine`` values Python reports there. The two ARM
spellings are not
+# synonyms: ``platform.machine()`` returns ``aarch64`` on Linux and ``arm64``
on macOS, so
+# each platform excludes exactly one of them. A provider whose native
dependency is
+# unavailable on every ARM target lists both platforms; one that only lacks a
Linux ARM
+# build lists ``linux/arm64`` alone and stays installable on Apple Silicon.
+#
+# Only the ``linux/*`` entries correspond to CI matrix platforms (see
``CI_PLATFORMS``);
+# ``darwin/arm64`` never matches a CI run and exists purely to drive the
install-time
+# marker.
EXCLUDED_PLATFORM_MACHINES: dict[str, list[str]] = {
- "linux/arm64": ["aarch64", "arm64"],
+ "linux/arm64": ["aarch64"],
+ "darwin/arm64": ["arm64"],
}
GITHUB_TOKEN_ENV_VARS = ("GH_TOKEN", "GITHUB_TOKEN")
diff --git a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
index ec59e21a036..7a6e7c26f24 100644
--- a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
+++ b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
@@ -17,7 +17,7 @@
from __future__ import annotations
import pytest
-from check_excluded_provider_markers import _check_dependency
+from check_excluded_provider_markers import _check_dependency,
_get_excluded_providers
def _excluded(python=None, machines=None):
@@ -115,3 +115,20 @@ class TestCheckDependencyPlatform:
'apache-airflow-providers-ibm-mq>=0.1.0; python_version !="3.14"
and platform_machine !="aarch64"'
)
assert _check_dependency(dep, excluded) == []
+
+
[email protected](scope="module")
+def excluded():
+ return _get_excluded_providers()
+
+
+class TestExcludedPlatformDerivation:
+ """``platform.machine()`` reports ``aarch64`` on Linux and ``arm64`` on
macOS, so each
+ excluded platform must contribute only its own machine spelling. A
provider that merely
+ lacks a Linux ARM build has to stay installable on Apple Silicon."""
+
+ def test_linux_arm_exclusion_leaves_apple_silicon_installable(self,
excluded):
+ assert excluded["apache-airflow-providers-ibm-db2"]["machines"] ==
["aarch64"]
+
+ def test_listing_both_arm_platforms_excludes_both_machines(self, excluded):
+ assert excluded["apache-airflow-providers-ibm-mq"]["machines"] ==
["aarch64", "arm64"]
diff --git a/uv.lock b/uv.lock
index 243691d0e61..d5a35a4b581 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6224,8 +6224,8 @@ dependencies = [
{ name = "apache-airflow" },
{ name = "apache-airflow-providers-common-compat" },
{ name = "apache-airflow-providers-common-sql" },
- { name = "ibm-db" },
- { name = "ibm-db-sa" },
+ { name = "ibm-db", marker = "platform_machine != 'aarch64'" },
+ { name = "ibm-db-sa", marker = "platform_machine != 'aarch64'" },
{ name = "methodtools" },
]
@@ -6251,8 +6251,8 @@ requires-dist = [
{ name = "apache-airflow-providers-common-compat", editable =
"providers/common/compat" },
{ name = "apache-airflow-providers-common-sql", editable =
"providers/common/sql" },
{ name = "apache-airflow-providers-openlineage", marker = "extra ==
'openlineage'", editable = "providers/openlineage" },
- { name = "ibm-db", specifier = ">=3.2.0" },
- { name = "ibm-db-sa", specifier = ">=0.4.0" },
+ { name = "ibm-db", marker = "platform_machine != 'aarch64'", specifier =
">=3.2.0" },
+ { name = "ibm-db-sa", marker = "platform_machine != 'aarch64'", specifier
= ">=0.4.0" },
{ name = "methodtools", specifier = ">=0.4.7" },
]
provides-extras = ["openlineage"]
