This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a1b02e26f3d Support Python patch exclusions in provider metadata
(#73598)
a1b02e26f3d is described below
commit a1b02e26f3da82d2b0b3484c8f5df5e7f1fe49bb
Author: Dev-iL <[email protected]>
AuthorDate: Sat Oct 3 23:20:15 2026 +0300
Support Python patch exclusions in provider metadata (#73598)
* Support patch-level Python floors for providers
Providers whose dependencies require a patch release within Airflow’s
lowest supported Python minor cannot express that constraint today. Excluding
the whole minor raises uv’s workspace floor and drops otherwise supported
interpreters, while omitting the marker makes the workspace unsatisfiable.
Compatibility exceptions also need an explicit rationale so future
contributors can distinguish intentional support gaps from stale metadata.
Generated-by: Codex (GPT-5)
* Allow provider Python patch exclusions
A provider dependency can require a later patch within a supported Python
minor. The existing minor exclusion removes too much, while a plain
exact-version exclusion leaves an unsatisfiable uv range.
---
airflow-core/src/airflow/provider.yaml.schema.json | 5 +-
.../tests/unit/always/test_provider_yaml_schema.py | 10 ++++
dev/breeze/tests/test_packages.py | 27 ++++++++++
providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 20 ++++++++
scripts/ci/prek/check_excluded_provider_markers.py | 36 ++++++++++++--
scripts/ci/prek/update_airflow_pyproject_toml.py | 35 +++++++------
.../prek/test_check_excluded_provider_markers.py | 56 +++++++++++++++++++++
.../ci/prek/test_update_airflow_pyproject_toml.py | 58 ++++++++++++++++++++++
8 files changed, 224 insertions(+), 23 deletions(-)
diff --git a/airflow-core/src/airflow/provider.yaml.schema.json
b/airflow-core/src/airflow/provider.yaml.schema.json
index 4c4a199a882..e53c0f0aeea 100644
--- a/airflow-core/src/airflow/provider.yaml.schema.json
+++ b/airflow-core/src/airflow/provider.yaml.schema.json
@@ -52,10 +52,11 @@
]
},
"excluded-python-versions": {
- "description": "List of python versions excluded for that
provider",
+ "description": "Python X.Y minors or X.Y.Z patches on which the
provider is unavailable. A minor is removed from the provider test matrix; a
patch keeps the minor's test lane. Generated provider requirements exclude
either version with a .* suffix. Root package extras use python_version markers
for minors and python_full_version markers for patches.",
"type": "array",
"items": {
- "type": "string"
+ "type": "string",
+ "pattern": "^[0-9]+\\.[0-9]+(\\.[0-9]+)?(?![\\s\\S])"
}
},
"excluded-platforms": {
diff --git a/airflow-core/tests/unit/always/test_provider_yaml_schema.py
b/airflow-core/tests/unit/always/test_provider_yaml_schema.py
index 70316be79e3..555d27aa241 100644
--- a/airflow-core/tests/unit/always/test_provider_yaml_schema.py
+++ b/airflow-core/tests/unit/always/test_provider_yaml_schema.py
@@ -41,6 +41,16 @@ CONNECTION_TYPE_SCHEMA =
json.loads(SCHEMA_PATH.read_text())["properties"]["conn
BLOCK_SCALAR_VALUE = yaml.safe_load("connection-type: |\n
pydanticai_vertex\n")["connection-type"]
[email protected]("invalid_version", ["3.10.", "3.10.0.1", "3.10.0\n"])
+def test_excluded_python_versions_accept_minor_or_patch_only(invalid_version):
+ schema =
json.loads(SCHEMA_PATH.read_text())["properties"]["excluded-python-versions"]["items"]
+
+ jsonschema.validate("3.10", schema=schema)
+ jsonschema.validate("3.10.0", schema=schema)
+ with pytest.raises(jsonschema.ValidationError):
+ jsonschema.validate(invalid_version, schema=schema)
+
+
@pytest.mark.parametrize(
"connection_type",
["pydanticai", "google_cloud_platform", "pagerduty_events", "a", "s3"],
diff --git a/dev/breeze/tests/test_packages.py
b/dev/breeze/tests/test_packages.py
index 2db1df22eb6..9c0899b5019 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -21,8 +21,11 @@ from pathlib import Path
from unittest import mock
import pytest
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
from airflow_breeze.global_constants import REGULAR_DOC_PACKAGES
+from airflow_breeze.utils import packages
from airflow_breeze.utils.packages import (
PipRequirements,
_process_line_with_next_version_comment,
@@ -272,6 +275,30 @@ def test_get_min_airflow_version(provider_id: str,
min_version: str):
assert get_min_airflow_version(provider_id) == min_version
+def test_patch_exclusion_in_generated_provider_metadata(monkeypatch):
+ details =
get_provider_details("asana")._replace(excluded_python_versions=["3.10.0",
"3.14"])
+ monkeypatch.setattr(packages, "get_provider_details", lambda provider_id:
details)
+
+ requirements = (
+ packages.get_python_requires("asana"),
+ get_provider_jinja_context("asana", current_release_version="1.0.0",
version_suffix="")[
+ "REQUIRES_PYTHON"
+ ],
+ )
+ for requirement in requirements:
+ assert "!=3.10.0.*" in requirement
+ specifier = SpecifierSet(requirement)
+ assert Version("3.10.0") not in specifier
+ assert Version("3.10.0.post1") not in specifier
+ assert Version("3.10.1") in specifier
+ assert Version("3.11.0") in specifier
+ assert Version("3.14.1") not in specifier
+
+ context = get_provider_jinja_context("asana",
current_release_version="1.0.0", version_suffix="")
+ assert "3.10" in context["SUPPORTED_PYTHON_VERSIONS"]
+ assert "3.14" not in context["SUPPORTED_PYTHON_VERSIONS"]
+
+
@pytest.mark.parametrize(
("cross_provider_deps", "suspended_ids", "requirements", "expected"),
[
diff --git a/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
b/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
index 08b000ea59b..070136b51db 100644
--- a/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
+++ b/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
@@ -272,6 +272,26 @@ your provider:
- hook-class-name:
airflow.providers.<PROVIDER>.hooks.<PROVIDER>.NewProviderHook
connection-type: provider_connection_type
+Provider compatibility exceptions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Two optional ``provider.yaml`` fields describe environments where a provider
cannot be installed:
+
+* ``excluded-python-versions`` accepts Python minors as ``X.Y`` and patches as
``X.Y.Z``.
+ The entry ``"3.14"`` excludes the whole minor from generated provider
``Requires-Python`` metadata.
+ Root extras use a ``python_version`` marker, and CI omits the provider from
the ``3.14`` test lane.
+ The entry ``"3.10.0"`` excludes only that patch prefix from provider
metadata.
+ Root extras use ``python_full_version``. CI keeps the ``3.10`` test lane.
+ Use a patch entry when a dependency needs ``>=3.10.1`` but the provider
otherwise supports Python 3.10.
+ The generated requirement uses ``!=3.10.0.*``. An exact ``!=3.10.0`` leaves
versions below ``3.10.1``
+ that uv cannot resolve.
+* ``excluded-platforms`` removes the provider from matching platform test
matrices and adds
+ ``platform_machine`` markers that prevent the root package extras from
installing it there.
+
+uv resolves all workspace members against their combined ``Requires-Python``
range. A patch excluded
+by a provider is therefore unavailable to the whole workspace during locking,
even though the root
+extra marker omits that provider on the patch.
+
Building documentation locally
-------------------------------
diff --git a/scripts/ci/prek/check_excluded_provider_markers.py
b/scripts/ci/prek/check_excluded_provider_markers.py
index 950b9faada5..076940419e6 100644
--- a/scripts/ci/prek/check_excluded_provider_markers.py
+++ b/scripts/ci/prek/check_excluded_provider_markers.py
@@ -31,8 +31,9 @@ the correct environment marker.
Provider exclusions are authoritative in each provider's ``provider.yaml``
(``excluded-python-versions`` and ``excluded-platforms`` fields). Any
dependency
string in the meta-package ``pyproject.toml`` that names an excluded provider
without
-a matching ``python_version != "X.Y"`` (per excluded Python version) or
-``platform_machine != "MACHINE"`` (per excluded platform) marker is flagged as
an error.
+a matching ``python_version != "X.Y"`` or ``python_full_version != "X.Y.Z.*"``
+(per excluded Python version) or ``platform_machine != "MACHINE"``
+(per excluded platform) marker is flagged as an error.
"""
from __future__ import annotations
@@ -93,11 +94,34 @@ def _check_dependency(dep_str: str, excluded_providers:
dict[str, dict[str, list
errors = []
exclusions = excluded_providers[package_name]
for version in exclusions.get("python", []):
- env = {"python_version": version}
- if req.marker is None or req.marker.evaluate(env):
+ parts = version.split(".")
+ is_patch = len(parts) == 3
+ marker_name = "python_full_version" if is_patch else "python_version"
+ marker_version = f"{version}.*" if is_patch else version
+ env = {
+ "python_version": ".".join(parts[:2]),
+ "python_full_version": version if is_patch else f"{version}.0",
+ "platform_machine": "__other__",
+ }
+ excluded_environments = [env]
+ if is_patch:
+ excluded_environments.append({**env, "python_full_version":
f"{version}.post1"})
+ if req.marker is None or any(req.marker.evaluate(excluded) for
excluded in excluded_environments):
errors.append(
- f'Dependency on "{package_name}" is missing python_version
!="{version}" marker: {dep_str}'
+ f'Dependency on "{package_name}" is missing {marker_name}
!="{marker_version}" marker: {dep_str}'
)
+ if is_patch and req.marker is not None:
+ adjacent_patches = [int(parts[2]) + 1]
+ if int(parts[2]):
+ adjacent_patches.append(int(parts[2]) - 1)
+ for patch in adjacent_patches:
+ adjacent = f"{parts[0]}.{parts[1]}.{patch}"
+ if adjacent in exclusions["python"] or ".".join(parts[:2]) in
exclusions["python"]:
+ continue
+ if not req.marker.evaluate({**env, "python_full_version":
adjacent}):
+ errors.append(
+ f'Dependency on "{package_name}" excludes supported
Python {adjacent}: {dep_str}'
+ )
for machine in exclusions.get("machines", []):
env = {"platform_machine": machine}
if req.marker is None or req.marker.evaluate(env):
@@ -141,6 +165,8 @@ def main() -> int:
"\n[yellow]Each dependency on a provider with
excluded-python-versions or "
"excluded-platforms in provider.yaml must have a matching
marker.[/]\n"
"[yellow]Example: 'apache-airflow-providers-amazon>=9.0.0;
python_version !=\"3.14\"'[/]\n"
+ "[yellow]Example: 'apache-airflow-providers-example>=1.0.0; "
+ 'python_full_version !="3.10.0.*"\'[/]\n'
'[yellow]Example: \'apache-airflow-providers-ibm-mq>=0.1.0;
platform_machine !="aarch64" '
'and platform_machine !="arm64"\'[/]'
)
diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py
b/scripts/ci/prek/update_airflow_pyproject_toml.py
index f66a3d7ba5d..a798bc7c915 100755
--- a/scripts/ci/prek/update_airflow_pyproject_toml.py
+++ b/scripts/ci/prek/update_airflow_pyproject_toml.py
@@ -120,10 +120,6 @@ PROVIDER_METADATA_FILE_PATH = AIRFLOW_ROOT_PATH /
"generated" / "provider_metada
PROVIDER_DEPENDENCIES_FILE_PATH = AIRFLOW_ROOT_PATH / "generated" /
"provider_dependencies.json"
file_list = sys.argv[1:]
-console.print("[bright_blue]Updating min-provider versions in
apache-airflow\n")
-
-all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
-all_providers_dependencies =
json.loads(PROVIDER_DEPENDENCIES_FILE_PATH.read_text())
def _read_toml(path: Path) -> dict[str, Any]:
@@ -179,9 +175,11 @@ def _fallback_provider_version(
return None, ""
-def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
+def find_min_provider_version(
+ provider_id: str, provider_metadata: dict[str, Any]
+) -> tuple[Version | None, str]:
console.print(f"[bright_blue]Finding min version for provider id:[/]
{provider_id}")
- metadata = all_providers_metadata.get(provider_id)
+ metadata = provider_metadata.get(provider_id)
# We should periodically update the starting date to avoid pip install
resolution issues
# TODO: when min Python version is 3.11 change back the code to
fromisoformat
# https://github.com/apache/airflow/pull/49155/files
@@ -236,20 +234,21 @@ PROVIDER_MIN_VERSIONS: dict[str, str | None] = {}
def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
"""
- Return an environment marker string excluding Python versions and
platforms.
+ Return an environment marker string for a provider's supported
environments.
- Combines ``excluded-python-versions`` and ``excluded-platforms`` from the
provider
- metadata into a single PEP 508 marker, e.g.:
- '; python_version != "3.14" and platform_machine != "aarch64" and
platform_machine != "arm64"'
+ Combines ``excluded-python-versions`` and ``excluded-platforms`` from the
+ provider metadata into a single PEP 508 marker.
- If neither is set, it returns an empty str.
+ If none is set, it returns an empty str.
"""
if not provider_dependencies:
return ""
- conditions = [
- f'python_version !=\\"{version}\\"'
- for version in provider_dependencies.get("excluded-python-versions",
[])
- ]
+ conditions = []
+ for version in provider_dependencies.get("excluded-python-versions", []):
+ if version.count(".") == 2:
+ conditions.append(f'python_full_version !=\\"{version}.*\\"')
+ else:
+ conditions.append(f'python_version !=\\"{version}\\"')
for platform in provider_dependencies.get("excluded-platforms", []):
conditions.extend(
f'platform_machine !=\\"{machine}\\"' for machine in
EXCLUDED_PLATFORM_MACHINES.get(platform, [])
@@ -260,6 +259,10 @@ def get_exclusion_marker(provider_dependencies: dict[str,
Any]) -> str:
if __name__ == "__main__":
+ console.print("[bright_blue]Updating min-provider versions in
apache-airflow\n")
+ all_providers_metadata =
json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
+ all_providers_dependencies =
json.loads(PROVIDER_DEPENDENCIES_FILE_PATH.read_text())
+
all_optional_dependencies = []
optional_airflow_core_dependencies =
get_optional_dependencies(AIRFLOW_CORE_PYPROJECT_TOML_FILE)
for optional in sorted(optional_airflow_core_dependencies):
@@ -284,7 +287,7 @@ if __name__ == "__main__":
all_provider_lines = []
for provider_id in released_providers:
distribution_name = provider_distribution_name(provider_id)
- min_provider_version, comment = find_min_provider_version(provider_id)
+ min_provider_version, comment = find_min_provider_version(provider_id,
all_providers_metadata)
exclusion_marker =
get_exclusion_marker(all_providers_dependencies.get(provider_id, {}))
if min_provider_version:
diff --git a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
index 7a6e7c26f24..e29b4087cba 100644
--- a/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
+++ b/scripts/tests/ci/prek/test_check_excluded_provider_markers.py
@@ -117,6 +117,62 @@ class TestCheckDependencyPlatform:
assert _check_dependency(dep, excluded) == []
[email protected](
+ ("version", "marker", "error"),
+ [
+ ("3.10.0", "", 'missing python_full_version !="3.10.0.*"'),
+ ("3.10.0", 'python_full_version != "3.10.1.*"', 'missing
python_full_version !="3.10.0.*"'),
+ ("3.10.0", 'python_full_version != "3.10.0"', 'missing
python_full_version !="3.10.0.*"'),
+ ("3.10.0", 'python_version != "3.10"', "excludes supported Python
3.10.1"),
+ (
+ "3.10.1",
+ 'python_full_version != "3.10.0.*" and python_full_version !=
"3.10.1.*"',
+ "excludes supported Python 3.10.0",
+ ),
+ ("3.10.0", 'python_full_version != "3.10.0.*"', None),
+ ("3.10.1", 'python_full_version != "3.10.1.*"', None),
+ ("3.11.0", 'python_full_version != "3.11.0.*"', None),
+ ],
+)
+def test_patch_exclusion_marker(version, marker, error):
+ dependency = "apache-airflow-providers-example>=1.0.0"
+ if marker:
+ dependency += f"; {marker}"
+ excluded = {"apache-airflow-providers-example": {"python": [version],
"machines": []}}
+
+ errors = _check_dependency(dependency, excluded)
+
+ if error:
+ assert any(error in message and "apache-airflow-providers-example" in
message for message in errors)
+ else:
+ assert errors == []
+
+
+def test_patch_minor_and_platform_exclusions_together():
+ dependency = (
+ "apache-airflow-providers-example>=1.0.0; "
+ 'python_full_version != "3.10.0.*" and python_version != "3.14" '
+ 'and platform_machine != "arm64"'
+ )
+ excluded = {"apache-airflow-providers-example": {"python": ["3.10.0",
"3.14"], "machines": ["arm64"]}}
+
+ assert _check_dependency(dependency, excluded) == []
+
+
[email protected](
+ ("versions", "marker"),
+ [
+ (["3.10.0", "3.10.1"], 'python_full_version != "3.10.0.*" and
python_full_version != "3.10.1.*"'),
+ (["3.10.0", "3.10"], 'python_full_version != "3.10.0.*" and
python_version != "3.10"'),
+ ],
+)
+def test_adjacent_excluded_patches_do_not_count_as_supported(versions, marker):
+ dependency = f"apache-airflow-providers-example>=1.0.0; {marker}"
+ excluded = {"apache-airflow-providers-example": {"python": versions,
"machines": []}}
+
+ assert _check_dependency(dependency, excluded) == []
+
+
@pytest.fixture(scope="module")
def excluded():
return _get_excluded_providers()
diff --git a/scripts/tests/ci/prek/test_update_airflow_pyproject_toml.py
b/scripts/tests/ci/prek/test_update_airflow_pyproject_toml.py
new file mode 100644
index 00000000000..0db91953810
--- /dev/null
+++ b/scripts/tests/ci/prek/test_update_airflow_pyproject_toml.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from ci.prek.update_airflow_pyproject_toml import get_exclusion_marker
+from packaging.requirements import Requirement
+
+
[email protected]("version", ["3.10.0", "3.11.0"])
+def test_get_exclusion_marker_for_patch(version):
+ marker = get_exclusion_marker({"excluded-python-versions": [version]})
+
+ assert marker == f'; python_full_version !=\\"{version}.*\\"'
+ unescaped_marker = marker.replace("\\", "")
+ requirement =
Requirement(f"apache-airflow-providers-example>=1.0{unescaped_marker}")
+ assert not requirement.marker.evaluate({"python_full_version": version})
+ assert not requirement.marker.evaluate({"python_full_version":
f"{version}.post1"})
+ minor, patch = version.rsplit(".", 1)
+ assert requirement.marker.evaluate({"python_full_version":
f"{minor}.{int(patch) + 1}"})
+
+
+def test_get_exclusion_marker_combines_patch_minor_and_platform():
+ marker = get_exclusion_marker(
+ {
+ "excluded-python-versions": ["3.10.0", "3.14"],
+ "excluded-platforms": ["linux/arm64"],
+ }
+ )
+
+ assert 'python_full_version !=\\"3.10.0.*\\"' in marker
+ assert 'python_version !=\\"3.14\\"' in marker
+ assert 'platform_machine !=\\"aarch64\\"' in marker
+ unescaped_marker = marker.replace("\\", "")
+ requirement =
Requirement(f"apache-airflow-providers-example>=1.0{unescaped_marker}")
+ assert requirement.marker.evaluate(
+ {"python_version": "3.10", "python_full_version": "3.10.1",
"platform_machine": "x86_64"}
+ )
+ assert not requirement.marker.evaluate(
+ {"python_version": "3.14", "python_full_version": "3.14.1",
"platform_machine": "x86_64"}
+ )
+ assert not requirement.marker.evaluate(
+ {"python_version": "3.10", "python_full_version": "3.10.1",
"platform_machine": "aarch64"}
+ )
