This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch check-core-shim-targets-exist in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ed217b38ddfc714c8e542266bd38159b72dcb561 Author: Shahar Epstein <[email protected]> AuthorDate: Sat Nov 7 20:41:27 2026 +0200 Keep provider classes that core deprecation shims still re-export Released Airflow 3 cores re-export moved provider classes, such as airflow.sensors.time_delta.TimeDeltaSensorAsync. Removing one in a provider release breaks those imports on every supported core, and dropping the shim entry on main does not reach released cores. #76131 removed TimeDeltaSensorAsync this way, so it has been reverted. A static check now fails when a core shim points at a provider target that no longer exists, and the deprecation warning states when the class can actually go. --- .pre-commit-config.yaml | 10 ++ .../providers/standard/sensors/time_delta.py | 4 +- scripts/ci/prek/check_core_shim_targets_exist.py | 142 +++++++++++++++++++++ .../ci/prek/test_check_core_shim_targets_exist.py | 97 ++++++++++++++ 4 files changed, 251 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9d64b98875..bb513199e89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1161,6 +1161,16 @@ repos: language: python pass_filenames: false files: ^\.github/workflows/ci-amd\.yml$|^scripts/ci/prek/check_ci_gate_needs\.py$ + - id: check-core-shim-targets-exist + name: Check that provider targets of core deprecation shims exist + env: + PYTHONPYCACHEPREFIX: .build/pycache + description: Released cores re-export these provider classes, so removing one breaks user imports + entry: ./scripts/ci/prek/check_core_shim_targets_exist.py + language: python + pass_filenames: false + # A change that only deletes a provider module matches no file pattern. + always_run: true - id: check-auth-manager-mock-targets name: Patch SimpleAuthManager, not BaseAuthManager, for methods it overrides env: diff --git a/providers/standard/src/airflow/providers/standard/sensors/time_delta.py b/providers/standard/src/airflow/providers/standard/sensors/time_delta.py index 1295a829bb3..341e458ee31 100644 --- a/providers/standard/src/airflow/providers/standard/sensors/time_delta.py +++ b/providers/standard/src/airflow/providers/standard/sensors/time_delta.py @@ -156,7 +156,6 @@ class TimeDeltaSensor(BaseSensorOperator): return None -# TODO: Remove in the next major release @deprecated( "Use `TimeDeltaSensor` with `deferrable=True` instead", category=AirflowProviderDeprecationWarning ) @@ -169,7 +168,8 @@ class TimeDeltaSensorAsync(TimeDeltaSensor): def __init__(self, *, end_from_trigger: bool = False, delta, **kwargs) -> None: warnings.warn( - "TimeDeltaSensorAsync is deprecated and will be removed in a future version. Use `TimeDeltaSensor` with `deferrable=True` instead.", + "TimeDeltaSensorAsync is deprecated and will be removed once this provider requires Airflow 4. " + "Use `TimeDeltaSensor` with `deferrable=True` instead.", AirflowProviderDeprecationWarning, stacklevel=2, ) diff --git a/scripts/ci/prek/check_core_shim_targets_exist.py b/scripts/ci/prek/check_core_shim_targets_exist.py new file mode 100755 index 00000000000..713e1aff634 --- /dev/null +++ b/scripts/ci/prek/check_core_shim_targets_exist.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11,<3.12" +# /// +"""Check that every provider target of a core ``add_deprecated_classes`` shim still exists. + +Core re-exports moved classes, such as ``airflow.sensors.time_delta.TimeDeltaSensorAsync``, from +providers. Released cores keep those shims, so a provider that removes a target breaks user +imports on every supported core that ships the shim, even if main drops the shim entry too. +A target can only go once the provider requires a core that no longer ships its shim. +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +AIRFLOW_ROOT = Path(__file__).parents[3] +CORE_SRC = AIRFLOW_ROOT / "airflow-core" / "src" +PROVIDERS_ROOT = AIRFLOW_ROOT / "providers" +PROVIDER_PREFIX = "airflow.providers." + + +def find_shim_files(core_src: Path = CORE_SRC) -> list[Path]: + return sorted( + path + for path in core_src.rglob("*.py") + if path.name != "deprecation_tools.py" and "add_deprecated_classes(" in path.read_text() + ) + + +def find_provider_targets(path: Path) -> list[tuple[int, str, str]]: + """Return ``(line, key, target)`` for each shim mapping that points into a provider.""" + targets = [] + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if not isinstance(node, ast.Dict): + continue + for key, value in zip(node.keys, node.values): + if ( + isinstance(key, ast.Constant) + and isinstance(key.value, str) + and isinstance(value, ast.Constant) + and isinstance(value.value, str) + and value.value.startswith(PROVIDER_PREFIX) + ): + targets.append((value.lineno, key.value, value.value)) + return targets + + +def find_module_file(module: str, providers_root: Path = PROVIDERS_ROOT) -> Path | None: + parts = module.split(".") + # Provider folders nest like the package (providers/apache/hive), so try each prefix. + for end in range(3, len(parts) + 1): + src = providers_root.joinpath(*parts[2:end], "src") + if not src.is_dir(): + continue + package = src.joinpath(*parts) + for candidate in (package.with_suffix(".py"), package / "__init__.py"): + if candidate.is_file(): + return candidate + return None + + +def is_type_checking(test: ast.expr) -> bool: + return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or ( + isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING" + ) + + +def get_module_names(path: Path) -> set[str]: + """Return names bound at runtime at module level; ``TYPE_CHECKING``-only names are left out.""" + names: set[str] = set() + + def collect(body: list[ast.stmt]) -> None: + for node in body: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + names.add(node.name) + elif isinstance(node, ast.Assign): + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + elif isinstance(node, ast.AnnAssign) and node.value and isinstance(node.target, ast.Name): + names.add(node.target.id) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update(alias.asname or alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.If): + # Version-gated branches cannot be resolved statically, so either branch counts. + if not is_type_checking(node.test): + collect(node.body) + collect(node.orelse) + elif isinstance(node, ast.Try): + collect(node.body) + for handler in node.handlers: + collect(handler.body) + collect(node.orelse) + collect(node.finalbody) + + collect(ast.parse(path.read_text(), filename=str(path)).body) + return names + + +def is_target_present(key: str, target: str, providers_root: Path = PROVIDERS_ROOT) -> bool: + if key == "*" or find_module_file(target, providers_root): + return find_module_file(target, providers_root) is not None + module, name = target.rsplit(".", 1) + module_file = find_module_file(module, providers_root) + return module_file is not None and name in get_module_names(module_file) + + +def main() -> int: + errors = [] + for path in find_shim_files(): + for line, key, target in find_provider_targets(path): + if not is_target_present(key, target): + errors.append( + f"{path.relative_to(AIRFLOW_ROOT)}:{line}: core shim points at `{target}`, which does " + "not exist. Released Airflow cores ship this shim, so removing the target breaks user " + "imports on them, and removing the shim entry from main does not help. Keep the target " + "until the provider requires an Airflow version that no longer ships the shim." + ) + for error in errors: + print(error) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py b/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py new file mode 100644 index 00000000000..4a4fabd320f --- /dev/null +++ b/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import textwrap + +import pytest +from check_core_shim_targets_exist import find_provider_targets, is_target_present, main + +MODULE = "airflow.providers.acme.sensors.wait" + + [email protected] +def providers_root(tmp_path): + package = tmp_path / "acme" / "src" / "airflow" / "providers" / "acme" / "sensors" + package.mkdir(parents=True) + (package / "wait.py").write_text( + textwrap.dedent( + """ + from typing import TYPE_CHECKING + + from airflow.sdk import BaseSensorOperator as Base + + if TYPE_CHECKING: + from typing_only import TypeOnly + + annotated_only: int + + try: + from fast import Engine + except ImportError: + Engine = None + + class WaitSensor(Base): ... + + def helper(): ... + """ + ) + ) + return tmp_path + + +def test_find_provider_targets_skips_non_provider_targets(write_python_file): + path = write_python_file( + """ + __deprecated_classes = { + "wait": { + "WaitSensor": "airflow.providers.acme.sensors.wait.WaitSensor", + "BaseSensorOperator": "airflow.sdk.bases.sensor.BaseSensorOperator", + }, + "acme": {"*": "airflow.providers.acme.sensors"}, + } + """ + ) + + assert [(key, target) for _, key, target in find_provider_targets(path)] == [ + ("WaitSensor", "airflow.providers.acme.sensors.wait.WaitSensor"), + ("*", "airflow.providers.acme.sensors"), + ] + + [email protected]( + ("key", "target", "expected"), + [ + pytest.param("WaitSensor", f"{MODULE}.WaitSensor", True, id="class"), + pytest.param("helper", f"{MODULE}.helper", True, id="function"), + pytest.param("Base", f"{MODULE}.Base", True, id="import-alias"), + pytest.param("Engine", f"{MODULE}.Engine", True, id="name-inside-try"), + pytest.param("wait", MODULE, True, id="module"), + pytest.param("*", MODULE, True, id="wildcard"), + pytest.param("WaitSensorAsync", f"{MODULE}.WaitSensorAsync", False, id="removed-class"), + pytest.param("*", "airflow.providers.acme.sensors.gone", False, id="removed-module"), + pytest.param("TypeOnly", f"{MODULE}.TypeOnly", False, id="type-checking-only"), + pytest.param("annotated_only", f"{MODULE}.annotated_only", False, id="annotation-without-value"), + pytest.param("X", "airflow.providers.missing.hooks.X", False, id="missing-provider"), + ], +) +def test_is_target_present(providers_root, key, target, expected): + assert is_target_present(key, target, providers_root) is expected + + +def test_main_passes_on_repository(): + assert main() == 0
