This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 48ec002f4eb Fix test infrastructure for Python-version-excluded
providers (#63793)
48ec002f4eb is described below
commit 48ec002f4ebb7f868a7f3f789496b384e224cd24
Author: Dev-iL <[email protected]>
AuthorDate: Tue Mar 17 17:31:30 2026 +0200
Fix test infrastructure for Python-version-excluded providers (#63793)
* Skip provider tests when all test directories are excluded
When running Providers[google] or Providers[amazon] on Python 3.14,
generate_args_for_pytest removes the test folders for excluded
providers, but the skip check in _run_test only triggered when the
--ignore filter itself removed something. Since the folders were
already removed upstream, the guard condition was never met, leaving
pytest with only flags and no test directories — causing it to crash
on unrecognized custom arguments.
Remove the overly strict guard so the skip fires whenever no test
directories remain in the args.
* Fix PROD image docker tests for Python-version-excluded providers
The docker tests expected all providers from
prod_image_installed_providers.txt
to be present, but providers like google and amazon declare
excluded-python-versions in their provider.yaml. On Python 3.14, these
providers are correctly excluded from the PROD image at build time, but
the tests didn't account for this.
Read provider.yaml exclusions and filter expected providers/imports based
on the Docker image's Python version.
* Skip Python-incompatible provider wheels during PROD image build
get_distribution_specs.py now reads Requires-Python metadata from each
wheel and skips wheels that are incompatible with the running
interpreter. This prevents excluded providers (e.g. amazon on 3.14)
from being passed to pip/uv and installed despite their exclusion.
Also fix the requires-python specifier generation in packages.py:
!=3.14 per PEP 440 only excludes 3.14.0, not 3.14.2. Use !=3.14.*
wildcard to exclude the entire minor version.
---
Dockerfile | 26 ++++
.../airflow_breeze/commands/testing_commands.py | 5 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 2 +-
docker-tests/tests/docker_tests/test_prod_image.py | 59 ++++++--
scripts/docker/get_distribution_specs.py | 26 ++++
scripts/tests/docker/__init__.py | 16 +++
.../tests/docker/test_get_distribution_specs.py | 152 +++++++++++++++++++++
7 files changed, 273 insertions(+), 13 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index ff6d27cbd35..c7d0b8c9f18 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1114,8 +1114,11 @@ from __future__ import annotations
import os
import sys
+import zipfile
+from email.parser import HeaderParser
from pathlib import Path
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
from packaging.utils import (
InvalidSdistFilename,
InvalidWheelFilename,
@@ -1123,6 +1126,23 @@ from packaging.utils import (
parse_wheel_filename,
)
+_CURRENT_PYTHON =
f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+
+
+def _compatible_with_current_python(wheel_path: str) -> bool:
+ """Return False if the wheel's Requires-Python excludes the running
interpreter."""
+ try:
+ with zipfile.ZipFile(wheel_path) as zf:
+ for name in zf.namelist():
+ if name.endswith(".dist-info/METADATA"):
+ requires =
HeaderParser().parsestr(zf.read(name).decode("utf-8")).get("Requires-Python")
+ if requires:
+ return _CURRENT_PYTHON in SpecifierSet(requires)
+ return True
+ except (zipfile.BadZipFile, InvalidSpecifier, KeyError) as exc:
+ print(f"Warning: could not check Requires-Python for {wheel_path}:
{exc}", file=sys.stderr)
+ return True
+
def print_package_specs(extras: str = "") -> None:
for package_path in sys.argv[1:]:
@@ -1134,6 +1154,12 @@ def print_package_specs(extras: str = "") -> None:
except InvalidSdistFilename:
print(f"Could not parse package name from {package_path}",
file=sys.stderr)
continue
+ if package_path.endswith(".whl") and not
_compatible_with_current_python(package_path):
+ print(
+ f"Skipping {package} (Requires-Python not satisfied by
{_CURRENT_PYTHON})",
+ file=sys.stderr,
+ )
+ continue
print(f"{package}{extras} @ file://{package_path}")
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index 391e3736fdd..c2b63ed378f 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -249,10 +249,9 @@ def _run_test(
pytest_args.extend(extra_pytest_args)
# Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument
# Which might be the case if we are ignoring some providers during
compatibility checks
- pytest_args_before_skip = pytest_args
pytest_args = [arg for arg in pytest_args if f"--ignore={arg}" not in
pytest_args]
- # Double check: If no test is leftover we can skip running the test
- if pytest_args_before_skip != pytest_args and
pytest_args[0].startswith("--"):
+ # If no test directory is left (all positional args were
excluded/ignored), skip
+ if pytest_args and pytest_args[0].startswith("--"):
return 0, f"Skipped test, no tests needed: {shell_params.test_type}"
run_cmd.extend(pytest_args)
try:
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index cfdc98447a6..8358cfcdc00 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -705,7 +705,7 @@ def get_provider_jinja_context(
requires_python_version: str = f">={DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
# Most providers require the same python versions, but some may have
exclusions
for excluded_python_version in provider_details.excluded_python_versions:
- requires_python_version += f",!={excluded_python_version}"
+ requires_python_version += f",!={excluded_python_version}.*"
context: dict[str, Any] = {
"PROVIDER_ID": provider_details.provider_id,
diff --git a/docker-tests/tests/docker_tests/test_prod_image.py
b/docker-tests/tests/docker_tests/test_prod_image.py
index 64f56f799ae..585f6a39d60 100644
--- a/docker-tests/tests/docker_tests/test_prod_image.py
+++ b/docker-tests/tests/docker_tests/test_prod_image.py
@@ -21,6 +21,7 @@ import os
from importlib.util import find_spec
import pytest
+import yaml
from python_on_whales import DockerException
from docker_tests.constants import AIRFLOW_ROOT_PATH
@@ -57,6 +58,39 @@ REGULAR_IMAGE_PROVIDERS = [
testing_slim_image = os.environ.get("TEST_SLIM_IMAGE", str(False)).lower() in
("true", "1", "yes")
+def _get_provider_python_exclusions() -> dict[str, list[str]]:
+ """Return mapping of provider_id -> list of excluded Python minor versions
from provider.yaml."""
+ exclusions: dict[str, list[str]] = {}
+ providers_root = AIRFLOW_ROOT_PATH / "providers"
+ for line in PROD_IMAGE_PROVIDERS_FILE_PATH.read_text().splitlines():
+ provider_id = line.split(">=")[0].strip()
+ if not provider_id or provider_id.startswith("#"):
+ continue
+ provider_yaml_path = providers_root / provider_id.replace(".", "/") /
"provider.yaml"
+ if not provider_yaml_path.exists():
+ continue
+ with open(provider_yaml_path) as f:
+ data = yaml.safe_load(f)
+ excluded = data.get("excluded-python-versions", [])
+ if excluded:
+ exclusions[provider_id] = [str(v) for v in excluded]
+ return exclusions
+
+
+PROVIDER_PYTHON_EXCLUSIONS = _get_provider_python_exclusions()
+
+
+def _get_python_minor_version(default_docker_image: str) -> str:
+ """Get Python minor version (e.g. '3.14') from the Docker image."""
+ python_version = run_bash_in_docker("python --version",
image=default_docker_image)
+ return ".".join(python_version.strip().split()[1].split(".")[:2])
+
+
+def _get_excluded_provider_ids(python_minor: str) -> set[str]:
+ """Return set of provider IDs excluded for the given Python minor
version."""
+ return {pid for pid, versions in PROVIDER_PYTHON_EXCLUSIONS.items() if
python_minor in versions}
+
+
class TestCommands:
def test_without_command(self, default_docker_image):
"""Checking the image without a command. It should return non-zero
exit code."""
@@ -91,7 +125,10 @@ class TestPythonPackages:
if testing_slim_image:
packages_to_install = set(SLIM_IMAGE_PROVIDERS)
else:
- packages_to_install = set(REGULAR_IMAGE_PROVIDERS)
+ python_minor = _get_python_minor_version(default_docker_image)
+ excluded_ids = _get_excluded_provider_ids(python_minor)
+ excluded_packages = {f"apache-airflow-providers-{pid.replace('.',
'-')}" for pid in excluded_ids}
+ packages_to_install = set(REGULAR_IMAGE_PROVIDERS) -
excluded_packages
assert len(packages_to_install) != 0
output = run_bash_in_docker(
"airflow providers list --output json",
@@ -197,15 +234,19 @@ class TestPythonPackages:
def test_check_dependencies_imports(
self, package_name: str, import_names: list[str],
default_docker_image: str
):
+ python_minor = _get_python_minor_version(default_docker_image)
+ excluded_ids = _get_excluded_provider_ids(python_minor)
+ # Skip individual provider test cases if the provider is excluded for
this Python version
+ if package_name in excluded_ids:
+ pytest.skip(f"Provider {package_name} is excluded for Python
{python_minor}")
if package_name == "providers":
- python_version = run_bash_in_docker(
- "python --version",
- image=default_docker_image,
- )
- if python_version.startswith("Python 3.13"):
- if "airflow.providers.fab" in import_names:
- import_names.remove("airflow.providers.fab")
- run_python_in_docker(f"import {','.join(import_names)}",
image=default_docker_image)
+ excluded_imports = {f"airflow.providers.{pid}" for pid in
excluded_ids}
+ import_names = [name for name in import_names if name not in
excluded_imports]
+ # FAB provider has import issues on Python 3.13
+ if python_minor == "3.13":
+ import_names = [name for name in import_names if name !=
"airflow.providers.fab"]
+ if import_names:
+ run_python_in_docker(f"import {','.join(import_names)}",
image=default_docker_image)
def test_there_is_no_opt_airflow_airflow_folder(self,
default_docker_image):
output = run_bash_in_docker(
diff --git a/scripts/docker/get_distribution_specs.py
b/scripts/docker/get_distribution_specs.py
index bff99a2c1cb..feb7f4cc37b 100755
--- a/scripts/docker/get_distribution_specs.py
+++ b/scripts/docker/get_distribution_specs.py
@@ -19,8 +19,11 @@ from __future__ import annotations
import os
import sys
+import zipfile
+from email.parser import HeaderParser
from pathlib import Path
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
from packaging.utils import (
InvalidSdistFilename,
InvalidWheelFilename,
@@ -28,6 +31,23 @@ from packaging.utils import (
parse_wheel_filename,
)
+_CURRENT_PYTHON =
f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+
+
+def _compatible_with_current_python(wheel_path: str) -> bool:
+ """Return False if the wheel's Requires-Python excludes the running
interpreter."""
+ try:
+ with zipfile.ZipFile(wheel_path) as zf:
+ for name in zf.namelist():
+ if name.endswith(".dist-info/METADATA"):
+ requires =
HeaderParser().parsestr(zf.read(name).decode("utf-8")).get("Requires-Python")
+ if requires:
+ return _CURRENT_PYTHON in SpecifierSet(requires)
+ return True
+ except (zipfile.BadZipFile, InvalidSpecifier, KeyError) as exc:
+ print(f"Warning: could not check Requires-Python for {wheel_path}:
{exc}", file=sys.stderr)
+ return True
+
def print_package_specs(extras: str = "") -> None:
for package_path in sys.argv[1:]:
@@ -39,6 +59,12 @@ def print_package_specs(extras: str = "") -> None:
except InvalidSdistFilename:
print(f"Could not parse package name from {package_path}",
file=sys.stderr)
continue
+ if package_path.endswith(".whl") and not
_compatible_with_current_python(package_path):
+ print(
+ f"Skipping {package} (Requires-Python not satisfied by
{_CURRENT_PYTHON})",
+ file=sys.stderr,
+ )
+ continue
print(f"{package}{extras} @ file://{package_path}")
diff --git a/scripts/tests/docker/__init__.py b/scripts/tests/docker/__init__.py
new file mode 100644
index 00000000000..13a83393a91
--- /dev/null
+++ b/scripts/tests/docker/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/scripts/tests/docker/test_get_distribution_specs.py
b/scripts/tests/docker/test_get_distribution_specs.py
new file mode 100644
index 00000000000..50a367f7166
--- /dev/null
+++ b/scripts/tests/docker/test_get_distribution_specs.py
@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+SCRIPT_PATH = Path(__file__).resolve().parents[2] / "docker" /
"get_distribution_specs.py"
+
+CURRENT_PYTHON_MAJOR_MINOR =
f"{sys.version_info.major}.{sys.version_info.minor}"
+
+
+def _make_wheel(directory: Path, name: str, version: str, requires_python: str
| None = None) -> Path:
+ """Create a minimal .whl (zip) with METADATA."""
+ safe_name = name.replace("-", "_")
+ wheel_path = directory / f"{safe_name}-{version}-py3-none-any.whl"
+ dist_info = f"{safe_name}-{version}.dist-info"
+ metadata_lines = [
+ "Metadata-Version: 2.1",
+ f"Name: {name}",
+ f"Version: {version}",
+ ]
+ if requires_python is not None:
+ metadata_lines.append(f"Requires-Python: {requires_python}")
+ with zipfile.ZipFile(wheel_path, "w") as zf:
+ zf.writestr(f"{dist_info}/METADATA", "\n".join(metadata_lines))
+ zf.writestr(f"{dist_info}/RECORD", "")
+ return wheel_path
+
+
+def _run_script(*wheel_paths: Path, env: dict[str, str] | None = None) ->
subprocess.CompletedProcess:
+ return subprocess.run(
+ [sys.executable, str(SCRIPT_PATH), *(str(p) for p in wheel_paths)],
+ capture_output=True,
+ text=True,
+ check=False,
+ env=env,
+ )
+
+
+class TestRequiresPythonFiltering:
+ def test_wheel_without_requires_python_is_included(self, tmp_path):
+ whl = _make_wheel(tmp_path, "some-package", "1.0.0")
+ result = _run_script(whl)
+ assert result.returncode == 0
+ assert f"some-package @ file://{whl}" in result.stdout
+
+ def test_wheel_matching_current_python_is_included(self, tmp_path):
+ whl = _make_wheel(tmp_path, "some-package", "1.0.0",
requires_python=">=3.10")
+ result = _run_script(whl)
+ assert result.returncode == 0
+ assert f"some-package @ file://{whl}" in result.stdout
+
+ def test_wheel_excluding_current_python_is_skipped(self, tmp_path):
+ whl = _make_wheel(
+ tmp_path,
+ "excluded-package",
+ "2.0.0",
+ requires_python=f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*",
+ )
+ result = _run_script(whl)
+ assert result.returncode == 0
+ assert result.stdout == ""
+ assert "Skipping" in result.stderr
+ assert "excluded-package" in result.stderr
+
+ def test_corrupt_wheel_is_included_with_warning(self, tmp_path):
+ bad_whl = tmp_path / "bad_package-1.0.0-py3-none-any.whl"
+ bad_whl.write_bytes(b"not a zip")
+ result = _run_script(bad_whl)
+ assert result.returncode == 0
+ assert f"bad-package @ file://{bad_whl}" in result.stdout
+ assert "Warning" in result.stderr
+
+
+class TestMixedInputs:
+ def test_compatible_and_incompatible_together(self, tmp_path):
+ good_whl = _make_wheel(
+ tmp_path, "apache-airflow-providers-standard", "1.0.0",
requires_python=">=3.10"
+ )
+ bad_whl = _make_wheel(
+ tmp_path,
+ "apache-airflow-providers-google",
+ "21.0.0",
+ requires_python=f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*",
+ )
+ result = _run_script(good_whl, bad_whl)
+ assert result.returncode == 0
+ assert "apache-airflow-providers-standard" in result.stdout
+ assert "apache-airflow-providers-google" not in result.stdout
+ assert "Skipping" in result.stderr
+
+ def test_sdist_is_not_filtered(self, tmp_path):
+ """Sdists cannot be inspected for Requires-Python without building, so
they pass through."""
+ import tarfile
+
+ sdist = tmp_path / "apache_airflow_providers_google-21.0.0.tar.gz"
+ with tarfile.open(sdist, "w:gz"):
+ pass
+ result = _run_script(sdist)
+ assert result.returncode == 0
+ assert "apache-airflow-providers-google" in result.stdout
+
+
+class TestExtrasEnvVar:
+ def test_extras_appended_to_spec(self, tmp_path):
+ import os
+
+ whl = _make_wheel(tmp_path, "apache-airflow", "3.0.0",
requires_python=">=3.10")
+ env = {**os.environ, "EXTRAS": "[celery,google]"}
+ result = _run_script(whl, env=env)
+ assert result.returncode == 0
+ assert f"apache-airflow[celery,google] @ file://{whl}" in result.stdout
+
+
[email protected](
+ ("requires_python", "should_include"),
+ [
+ pytest.param(">=3.10", True, id="lower-bound-satisfied"),
+ pytest.param(f"!={CURRENT_PYTHON_MAJOR_MINOR}.*", False,
id="wildcard-minor-excluded"),
+ pytest.param(f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*", False,
id="range-with-exclusion"),
+ pytest.param("<3.10", False, id="upper-bound-below-current"),
+ pytest.param(None, True, id="no-requires-python"),
+ ],
+)
+def test_requires_python_specifiers(tmp_path, requires_python, should_include):
+ whl = _make_wheel(tmp_path, "test-package", "1.0.0",
requires_python=requires_python)
+ result = _run_script(whl)
+ assert result.returncode == 0
+ if should_include:
+ assert f"test-package @ file://{whl}" in result.stdout
+ else:
+ assert result.stdout == ""
+ assert "Skipping" in result.stderr
