This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 98316545c92 Exclude ibm.mq provider from being installed on ARM
(#67783)
98316545c92 is described below
commit 98316545c924a19aae2f497a936b08a44a3b8ab3
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun May 31 11:05:31 2026 +0200
Exclude ibm.mq provider from being installed on ARM (#67783)
---
.pre-commit-config.yaml | 2 +-
providers/ibm/mq/pyproject.toml | 4 +-
pyproject.toml | 4 +-
scripts/ci/prek/check_excluded_provider_markers.py | 62 +++++++++++++++-------
scripts/ci/prek/common_prek_utils.py | 9 ++++
scripts/ci/prek/update_airflow_pyproject_toml.py | 39 +++++++++-----
.../prek/test_check_excluded_provider_markers.py | 44 +++++++++++++--
uv.lock | 12 ++---
8 files changed, 128 insertions(+), 48 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c084a48f152..f70c764d626 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -902,7 +902,7 @@ repos:
pass_filenames: false
require_serial: true
- id: check-excluded-provider-markers
- name: Check excluded-provider python_version markers in pyproject.toml
+ name: Check excluded-provider python_version and platform markers in
pyproject.toml
language: python
entry: ./scripts/ci/prek/check_excluded_provider_markers.py
files: >
diff --git a/providers/ibm/mq/pyproject.toml b/providers/ibm/mq/pyproject.toml
index b748d28e37f..c0e5e044112 100644
--- a/providers/ibm/mq/pyproject.toml
+++ b/providers/ibm/mq/pyproject.toml
@@ -70,7 +70,9 @@ dependencies = [
[project.optional-dependencies]
"ibmmq" = [
# Required at Runtime
- "ibmmq>=2.0.6",
+ # IBM MQ ships its native C client only for Linux x86_64 / Windows x64, so
the `ibmmq`
+ # bindings cannot be built on ARM. Skip it there (mirrors provider.yaml
excluded-platforms).
+ "ibmmq>=2.0.6; platform_machine != \"aarch64\" and platform_machine !=
\"arm64\"",
]
"common.messaging" = [
"apache-airflow-providers-common-messaging>=2.0.0"
diff --git a/pyproject.toml b/pyproject.toml
index 3821bce62c1..c63ef9fd447 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -249,7 +249,7 @@ apache-airflow = "airflow.__main__:main"
"apache-airflow-providers-http>=4.13.2"
]
"ibm.mq" = [
- "apache-airflow-providers-ibm-mq>=0.1.0"
+ "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\"
and platform_machine !=\"arm64\""
]
"imap" = [
"apache-airflow-providers-imap>=3.8.0"
@@ -454,7 +454,7 @@ apache-airflow = "airflow.__main__:main"
"apache-airflow-providers-grpc>=3.7.0",
"apache-airflow-providers-hashicorp>=4.0.0",
"apache-airflow-providers-http>=4.13.2",
- "apache-airflow-providers-ibm-mq>=0.1.0",
+ "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\"
and platform_machine !=\"arm64\"",
"apache-airflow-providers-imap>=3.8.0",
"apache-airflow-providers-influxdb>=2.8.0",
"apache-airflow-providers-informatica>=0.1.1",
diff --git a/scripts/ci/prek/check_excluded_provider_markers.py
b/scripts/ci/prek/check_excluded_provider_markers.py
index b3426980d02..950b9faada5 100644
--- a/scripts/ci/prek/check_excluded_provider_markers.py
+++ b/scripts/ci/prek/check_excluded_provider_markers.py
@@ -25,13 +25,14 @@
# ]
# ///
"""
-Validate that every dependency on a Python-version-excluded provider
-in pyproject.toml carries the correct ``python_version`` environment marker.
+Validate that every dependency on an excluded provider in pyproject.toml
carries
+the correct environment marker.
Provider exclusions are authoritative in each provider's ``provider.yaml``
-(``excluded-python-versions`` field). Any dependency string in the
-meta-package ``pyproject.toml`` that names an excluded provider without a
-matching ``python_version != "X.Y"`` marker is flagged as an error.
+(``excluded-python-versions`` and ``excluded-platforms`` fields). Any
dependency
+string in the meta-package ``pyproject.toml`` that names an excluded provider
without
+a matching ``python_version != "X.Y"`` (per excluded Python version) or
+``platform_machine != "MACHINE"`` (per excluded platform) marker is flagged as
an error.
"""
from __future__ import annotations
@@ -40,7 +41,7 @@ import sys
from pathlib import Path
import yaml
-from common_prek_utils import AIRFLOW_PROVIDERS_ROOT_PATH, AIRFLOW_ROOT_PATH
+from common_prek_utils import AIRFLOW_PROVIDERS_ROOT_PATH, AIRFLOW_ROOT_PATH,
EXCLUDED_PLATFORM_MACHINES
from packaging.requirements import InvalidRequirement, Requirement
from rich.console import Console
@@ -57,21 +58,30 @@ def _load_toml(path: Path) -> dict:
return tomllib.loads(path.read_text())
-def _get_excluded_providers() -> dict[str, list[str]]:
- """Return {normalized-package-name: [excluded python versions]} from
provider.yaml files."""
- excluded: dict[str, list[str]] = {}
+def _get_excluded_providers() -> dict[str, dict[str, list[str]]]:
+ """Return {normalized-package-name: {"python": [versions], "machines":
[machines]}}.
+
+ ``python`` holds the excluded Python versions; ``machines`` holds the
+ ``platform_machine`` values derived from the provider's excluded platforms.
+ """
+ excluded: dict[str, dict[str, list[str]]] = {}
for provider_yaml in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"):
if provider_yaml.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"):
continue
data = yaml.safe_load(provider_yaml.read_text())
- versions = data.get("excluded-python-versions", [])
- if versions:
+ versions = [str(v) for v in data.get("excluded-python-versions", [])]
+ machines = [
+ machine
+ for platform in data.get("excluded-platforms", [])
+ for machine in EXCLUDED_PLATFORM_MACHINES.get(platform, [])
+ ]
+ if versions or machines:
package_name = data["package-name"].lower().replace("_", "-")
- excluded[package_name] = [str(v) for v in versions]
+ excluded[package_name] = {"python": versions, "machines": machines}
return excluded
-def _check_dependency(dep_str: str, excluded_providers: dict[str, list[str]])
-> list[str]:
+def _check_dependency(dep_str: str, excluded_providers: dict[str, dict[str,
list[str]]]) -> list[str]:
"""Check a single dependency string. Return list of error messages."""
try:
req = Requirement(dep_str)
@@ -81,12 +91,19 @@ def _check_dependency(dep_str: str, excluded_providers:
dict[str, list[str]]) ->
if package_name not in excluded_providers:
return []
errors = []
- for version in excluded_providers[package_name]:
+ exclusions = excluded_providers[package_name]
+ for version in exclusions.get("python", []):
env = {"python_version": version}
if req.marker is None or req.marker.evaluate(env):
errors.append(
f'Dependency on "{package_name}" is missing python_version
!="{version}" marker: {dep_str}'
)
+ for machine in exclusions.get("machines", []):
+ env = {"platform_machine": machine}
+ if req.marker is None or req.marker.evaluate(env):
+ errors.append(
+ f'Dependency on "{package_name}" is missing platform_machine
!="{machine}" marker: {dep_str}'
+ )
return errors
@@ -96,8 +113,13 @@ def main() -> int:
return 0
console.print("[bright_blue]Checking excluded-provider markers in
pyproject.toml")
- for pkg, versions in sorted(excluded_providers.items()):
- console.print(f" [bright_blue]{pkg}[/] excluded for Python {',
'.join(versions)}")
+ for pkg, exclusions in sorted(excluded_providers.items()):
+ details = []
+ if exclusions["python"]:
+ details.append(f"Python {', '.join(exclusions['python'])}")
+ if exclusions["machines"]:
+ details.append(f"machine {', '.join(exclusions['machines'])}")
+ console.print(f" [bright_blue]{pkg}[/] excluded for {';
'.join(details)}")
toml_data = _load_toml(PYPROJECT_TOML_PATH)
all_errors: list[str] = []
@@ -116,9 +138,11 @@ def main() -> int:
for error in all_errors:
console.print(f" [red]✗[/] {error}")
console.print(
- "\n[yellow]Each dependency on a provider with
excluded-python-versions in "
- "provider.yaml must have a matching python_version marker.[/]\n"
- "[yellow]Example: 'apache-airflow-providers-amazon>=9.0.0;
python_version !=\"3.14\"'[/]"
+ "\n[yellow]Each dependency on a provider with
excluded-python-versions or "
+ "excluded-platforms in provider.yaml must have a matching
marker.[/]\n"
+ "[yellow]Example: 'apache-airflow-providers-amazon>=9.0.0;
python_version !=\"3.14\"'[/]\n"
+ '[yellow]Example: \'apache-airflow-providers-ibm-mq>=0.1.0;
platform_machine !="aarch64" '
+ 'and platform_machine !="arm64"\'[/]'
)
return 1
diff --git a/scripts/ci/prek/common_prek_utils.py
b/scripts/ci/prek/common_prek_utils.py
index 697fae98116..434bf13a482 100644
--- a/scripts/ci/prek/common_prek_utils.py
+++ b/scripts/ci/prek/common_prek_utils.py
@@ -45,6 +45,15 @@ KNOWN_SECOND_LEVEL_PATHS = ["apache", "atlassian", "common",
"cncf", "dbt", "mic
DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10"
+# Maps a Docker build platform string (as declared in ``provider.yaml`` under
+# ``excluded-platforms``) to the ``platform_machine`` values Python reports on
that
+# architecture. ``linux/arm64`` covers both Linux (``aarch64``) and macOS
Apple Silicon
+# (``arm64``) so a provider opting out of ARM is never pulled in on any ARM
machine where
+# its native dependency cannot be built.
+EXCLUDED_PLATFORM_MACHINES: dict[str, list[str]] = {
+ "linux/arm64": ["aarch64", "arm64"],
+}
+
GITHUB_TOKEN_ENV_VARS = ("GH_TOKEN", "GITHUB_TOKEN")
try:
diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py
b/scripts/ci/prek/update_airflow_pyproject_toml.py
index cb8afd4a341..a7d65331185 100755
--- a/scripts/ci/prek/update_airflow_pyproject_toml.py
+++ b/scripts/ci/prek/update_airflow_pyproject_toml.py
@@ -37,7 +37,13 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
-from common_prek_utils import AIRFLOW_ROOT_PATH, console,
get_all_provider_ids, insert_documentation
+from common_prek_utils import (
+ AIRFLOW_ROOT_PATH,
+ EXCLUDED_PLATFORM_MACHINES,
+ console,
+ get_all_provider_ids,
+ insert_documentation,
+)
from packaging.version import Version, parse as parse_version
AIRFLOW_PYPROJECT_TOML_FILE = AIRFLOW_ROOT_PATH / "pyproject.toml"
@@ -228,23 +234,28 @@ def find_min_provider_version(provider_id: str) ->
tuple[Version | None, str]:
PROVIDER_MIN_VERSIONS: dict[str, str | None] = {}
-def get_python_exclusion(provider_dependencies: dict[str, Any]) -> str:
+def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
"""
- Return a Python version exclusion marker string based on provider metadata.
+ Return an environment marker string excluding Python versions and
platforms.
- If there are excluded Python versions in the metadata, this function
returns a
- marker string like: '; python_version != "3.8" and python_version !=
"3.11"'
+ Combines ``excluded-python-versions`` and ``excluded-platforms`` from the
provider
+ metadata into a single PEP 508 marker, e.g.:
+ '; python_version != "3.14" and platform_machine != "aarch64" and
platform_machine != "arm64"'
- If none are found, it returns an empty str.
+ If neither is set, it returns an empty str.
"""
if not provider_dependencies:
return ""
- python_exclusions = provider_dependencies.get("excluded-python-versions",
[])
- if python_exclusions:
- python_exclusions_str = "and ".join(
- f'python_version !=\\"{version}\\"' for version in
python_exclusions
+ conditions = [
+ f'python_version !=\\"{version}\\"'
+ for version in provider_dependencies.get("excluded-python-versions",
[])
+ ]
+ for platform in provider_dependencies.get("excluded-platforms", []):
+ conditions.extend(
+ f'platform_machine !=\\"{machine}\\"' for machine in
EXCLUDED_PLATFORM_MACHINES.get(platform, [])
)
- return f"; {python_exclusions_str}"
+ if conditions:
+ return f"; {' and '.join(conditions)}"
return ""
@@ -263,14 +274,14 @@ if __name__ == "__main__":
for provider_id in all_providers:
distribution_name = provider_distribution_name(provider_id)
min_provider_version, comment = find_min_provider_version(provider_id)
- python_exclusion =
get_python_exclusion(all_providers_dependencies.get(provider_id, {}))
+ exclusion_marker =
get_exclusion_marker(all_providers_dependencies.get(provider_id, {}))
if min_provider_version:
all_provider_lines.append(
- f'
"{distribution_name}>={min_provider_version}{python_exclusion}",{comment}\n'
+ f'
"{distribution_name}>={min_provider_version}{exclusion_marker}",{comment}\n'
)
all_optional_dependencies.append(
- f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}{python_exclusion}"{comment}\n]\n'
+ f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}{exclusion_marker}"{comment}\n]\n'
)
else:
all_optional_dependencies.append(f'"{provider_id}" = [\n
"{distribution_name}"\n]\n')
diff --git a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
index c068367c06a..ec59e21a036 100644
--- a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
+++ b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
@@ -20,10 +20,14 @@ import pytest
from check_excluded_provider_markers import _check_dependency
+def _excluded(python=None, machines=None):
+ return {"python": list(python or []), "machines": list(machines or [])}
+
+
class TestCheckDependency:
EXCLUDED = {
- "apache-airflow-providers-amazon": ["3.14"],
- "apache-airflow-providers-google": ["3.14"],
+ "apache-airflow-providers-amazon": _excluded(python=["3.14"]),
+ "apache-airflow-providers-google": _excluded(python=["3.14"]),
}
def test_no_error_when_marker_present(self):
@@ -64,14 +68,14 @@ class TestCheckDependency:
],
)
def test_multiple_excluded_versions(self, excluded_versions):
- excluded = {"apache-airflow-providers-amazon": excluded_versions}
+ excluded = {"apache-airflow-providers-amazon":
_excluded(python=excluded_versions)}
dep = "apache-airflow-providers-amazon>=9.0.0"
errors = _check_dependency(dep, excluded)
assert len(errors) == len(excluded_versions)
def test_partial_marker_flags_missing_version(self):
"""If provider is excluded for 3.14 and 3.15, but only 3.14 marker
exists."""
- excluded = {"apache-airflow-providers-amazon": ["3.14", "3.15"]}
+ excluded = {"apache-airflow-providers-amazon":
_excluded(python=["3.14", "3.15"])}
dep = 'apache-airflow-providers-amazon>=9.0.0; python_version !="3.14"'
errors = _check_dependency(dep, excluded)
assert len(errors) == 1
@@ -79,5 +83,35 @@ class TestCheckDependency:
def test_and_combined_markers(self):
dep = 'apache-airflow-providers-amazon>=9.0.0; python_version !="3.14"
and python_version !="3.15"'
- excluded = {"apache-airflow-providers-amazon": ["3.14", "3.15"]}
+ excluded = {"apache-airflow-providers-amazon":
_excluded(python=["3.14", "3.15"])}
+ assert _check_dependency(dep, excluded) == []
+
+
+class TestCheckDependencyPlatform:
+ EXCLUDED = {"apache-airflow-providers-ibm-mq":
_excluded(machines=["aarch64", "arm64"])}
+
+ def test_no_error_when_platform_marker_present(self):
+ dep = (
+ 'apache-airflow-providers-ibm-mq>=0.1.0; platform_machine
!="aarch64" '
+ 'and platform_machine !="arm64"'
+ )
+ assert _check_dependency(dep, self.EXCLUDED) == []
+
+ def test_error_when_platform_marker_missing(self):
+ dep = "apache-airflow-providers-ibm-mq>=0.1.0"
+ errors = _check_dependency(dep, self.EXCLUDED)
+ assert len(errors) == 2
+ assert all("platform_machine" in e for e in errors)
+
+ def test_partial_platform_marker_flags_missing_machine(self):
+ dep = 'apache-airflow-providers-ibm-mq>=0.1.0; platform_machine
!="aarch64"'
+ errors = _check_dependency(dep, self.EXCLUDED)
+ assert len(errors) == 1
+ assert "arm64" in errors[0]
+
+ def test_combined_python_and_platform_exclusions(self):
+ excluded = {"apache-airflow-providers-ibm-mq":
_excluded(python=["3.14"], machines=["aarch64"])}
+ dep = (
+ 'apache-airflow-providers-ibm-mq>=0.1.0; python_version !="3.14"
and platform_machine !="aarch64"'
+ )
assert _check_dependency(dep, excluded) == []
diff --git a/uv.lock b/uv.lock
index eaf84b6de2c..a66b1726372 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1036,7 +1036,7 @@ all = [
{ name = "apache-airflow-providers-grpc" },
{ name = "apache-airflow-providers-hashicorp" },
{ name = "apache-airflow-providers-http" },
- { name = "apache-airflow-providers-ibm-mq" },
+ { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine !=
'aarch64' and platform_machine != 'arm64'" },
{ name = "apache-airflow-providers-imap" },
{ name = "apache-airflow-providers-influxdb" },
{ name = "apache-airflow-providers-informatica" },
@@ -1273,7 +1273,7 @@ http = [
{ name = "apache-airflow-providers-http" },
]
ibm-mq = [
- { name = "apache-airflow-providers-ibm-mq" },
+ { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine !=
'aarch64' and platform_machine != 'arm64'" },
]
imap = [
{ name = "apache-airflow-providers-imap" },
@@ -1649,8 +1649,8 @@ requires-dist = [
{ name = "apache-airflow-providers-hashicorp", marker = "extra ==
'hashicorp'", editable = "providers/hashicorp" },
{ name = "apache-airflow-providers-http", marker = "extra == 'all'",
editable = "providers/http" },
{ name = "apache-airflow-providers-http", marker = "extra == 'http'",
editable = "providers/http" },
- { name = "apache-airflow-providers-ibm-mq", marker = "extra == 'all'",
editable = "providers/ibm/mq" },
- { name = "apache-airflow-providers-ibm-mq", marker = "extra == 'ibm-mq'",
editable = "providers/ibm/mq" },
+ { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine !=
'aarch64' and platform_machine != 'arm64' and extra == 'all'", editable =
"providers/ibm/mq" },
+ { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine !=
'aarch64' and platform_machine != 'arm64' and extra == 'ibm-mq'", editable =
"providers/ibm/mq" },
{ name = "apache-airflow-providers-imap", marker = "extra == 'all'",
editable = "providers/imap" },
{ name = "apache-airflow-providers-imap", marker = "extra == 'imap'",
editable = "providers/imap" },
{ name = "apache-airflow-providers-influxdb", marker = "extra == 'all'",
editable = "providers/influxdb" },
@@ -5774,7 +5774,7 @@ common-messaging = [
{ name = "apache-airflow-providers-common-messaging" },
]
ibmmq = [
- { name = "ibmmq" },
+ { name = "ibmmq", marker = "platform_machine != 'aarch64' and
platform_machine != 'arm64'" },
]
[package.dev-dependencies]
@@ -5796,7 +5796,7 @@ requires-dist = [
{ name = "apache-airflow-providers-common-messaging", marker = "extra ==
'common-messaging'", editable = "providers/common/messaging" },
{ name = "asgiref", marker = "python_full_version < '3.14'", specifier =
">=2.3.0" },
{ name = "asgiref", marker = "python_full_version >= '3.14'", specifier =
">=3.11.1" },
- { name = "ibmmq", marker = "extra == 'ibmmq'", specifier = ">=2.0.6" },
+ { name = "ibmmq", marker = "platform_machine != 'aarch64' and
platform_machine != 'arm64' and extra == 'ibmmq'", specifier = ">=2.0.6" },
]
provides-extras = ["ibmmq", "common-messaging"]
