This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch check-core-shim-targets-exist in repository https://gitbox.apache.org/repos/asf/airflow.git
commit dbecc60cf3068f622db97b5a54e0370d312e9812 Author: Shahar Epstein <[email protected]> AuthorDate: Sat Nov 7 21:14:54 2026 +0200 Record core shim provider targets so they cannot be removed with their shim #76131 removed TimeDeltaSensorAsync and its core shim entry in one change. Checking only the shim maps on main cannot see that, because the entry is gone too, yet released cores still ship the shim. A committed list of the targets from main and every released core keeps them checked after main drops a shim, so removing one needs a deliberate edit to that list. --- scripts/ci/prek/check_core_shim_targets_exist.py | 50 +++++++++++++++---- scripts/ci/prek/core_shim_provider_targets.txt | 56 ++++++++++++++++++++++ .../ci/prek/test_check_core_shim_targets_exist.py | 47 +++++++++++++++++- 3 files changed, 144 insertions(+), 9 deletions(-) diff --git a/scripts/ci/prek/check_core_shim_targets_exist.py b/scripts/ci/prek/check_core_shim_targets_exist.py index 713e1aff634..12f204bc338 100755 --- a/scripts/ci/prek/check_core_shim_targets_exist.py +++ b/scripts/ci/prek/check_core_shim_targets_exist.py @@ -23,19 +23,25 @@ Core re-exports moved classes, such as ``airflow.sensors.time_delta.TimeDeltaSensorAsync``, from providers. Released cores keep those shims, so a provider that removes a target breaks user imports on every supported core that ships the shim, even if main drops the shim entry too. -A target can only go once the provider requires a core that no longer ships its shim. + +The targets are recorded in ``core_shim_provider_targets.txt``. Every provider target of a shim on +main must be recorded there, and every recorded target must exist. Removing a target together +with its shim entry therefore also needs a deliberate edit to that file. """ from __future__ import annotations import ast +import os import sys from pathlib import Path AIRFLOW_ROOT = Path(__file__).parents[3] CORE_SRC = AIRFLOW_ROOT / "airflow-core" / "src" PROVIDERS_ROOT = AIRFLOW_ROOT / "providers" +SNAPSHOT = Path(__file__).with_name("core_shim_provider_targets.txt") PROVIDER_PREFIX = "airflow.providers." +WILDCARD_SUFFIX = ".*" def find_shim_files(core_src: Path = CORE_SRC) -> list[Path]: @@ -122,17 +128,45 @@ def is_target_present(key: str, target: str, providers_root: Path = PROVIDERS_RO return module_file is not None and name in get_module_names(module_file) -def main() -> int: +def build_snapshot_entry(key: str, target: str) -> str: + return f"{target}{WILDCARD_SUFFIX}" if key == "*" else target + + +def read_snapshot(path: Path) -> dict[str, int]: + """Return each recorded target with its line number.""" + return { + line.strip(): number + for number, line in enumerate(path.read_text().splitlines(), start=1) + if line.strip() and not line.startswith("#") + } + + +def is_entry_present(entry: str, providers_root: Path = PROVIDERS_ROOT) -> bool: + if entry.endswith(WILDCARD_SUFFIX): + return is_target_present("*", entry.removesuffix(WILDCARD_SUFFIX), providers_root) + return is_target_present("", entry, providers_root) + + +def main(core_src: Path = CORE_SRC, providers_root: Path = PROVIDERS_ROOT, snapshot: Path = SNAPSHOT) -> int: + recorded = read_snapshot(snapshot) errors = [] - for path in find_shim_files(): + for path in find_shim_files(core_src): for line, key, target in find_provider_targets(path): - if not is_target_present(key, target): + entry = build_snapshot_entry(key, target) + if entry not in recorded: errors.append( - f"{path.relative_to(AIRFLOW_ROOT)}:{line}: core shim points at `{target}`, which does " - "not exist. Released Airflow cores ship this shim, so removing the target breaks user " - "imports on them, and removing the shim entry from main does not help. Keep the target " - "until the provider requires an Airflow version that no longer ships the shim." + f"{os.path.relpath(path)}:{line}: `{entry}` is missing from {os.path.relpath(snapshot)}. " + "Add it, so the target stays checked after main drops this shim." ) + if not is_entry_present(entry, providers_root): + errors.append(f"{os.path.relpath(path)}:{line}: `{entry}` does not exist.") + for entry, line in recorded.items(): + if not is_entry_present(entry, providers_root): + errors.append( + f"{os.path.relpath(snapshot)}:{line}: `{entry}` does not exist. Released Airflow cores " + "ship a shim for it, so removing it breaks user imports on them, even if main drops the " + "shim entry. Keep it until the provider requires an Airflow version without the shim." + ) for error in errors: print(error) return 1 if errors else 0 diff --git a/scripts/ci/prek/core_shim_provider_targets.txt b/scripts/ci/prek/core_shim_provider_targets.txt new file mode 100644 index 00000000000..2b13efd2656 --- /dev/null +++ b/scripts/ci/prek/core_shim_provider_targets.txt @@ -0,0 +1,56 @@ +# Provider targets of core ``add_deprecated_classes`` shims, from main and every released core. +# Released cores keep their shims, so a target stays here, and in its provider, after main +# drops the shim entry. Remove a line only when no provider version that ships the target +# supports a core that still has the shim, i.e. after the provider's Airflow floor passes it. +# +# Format: one dotted target per line; ``<module>.*`` is a wildcard (``"*"``) shim. +airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler +airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler +airflow.providers.common.sql.operators.generic_transfer.GenericTransfer +airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler +airflow.providers.google.cloud.log.gcs_task_handler.GCSTaskHandler +airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler +airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler +airflow.providers.smtp.operators.smtp.EmailOperator +airflow.providers.smtp.utils.emailer.build_mime_message +airflow.providers.smtp.utils.emailer.get_email_address_list +airflow.providers.smtp.utils.emailer.send_email +airflow.providers.smtp.utils.emailer.send_email_smtp +airflow.providers.smtp.utils.emailer.send_mime_email +airflow.providers.standard.hooks.filesystem.FSHook +airflow.providers.standard.hooks.package_index.PackageIndexHook +airflow.providers.standard.hooks.subprocess.SubprocessHook +airflow.providers.standard.operators.bash.BashOperator +airflow.providers.standard.operators.branch.BaseBranchOperator +airflow.providers.standard.operators.branch.BranchMixIn +airflow.providers.standard.operators.datetime.BranchDateTimeOperator +airflow.providers.standard.operators.empty.EmptyOperator +airflow.providers.standard.operators.latest_only.LatestOnlyOperator +airflow.providers.standard.operators.python.BranchExternalPythonOperator +airflow.providers.standard.operators.python.BranchPythonOperator +airflow.providers.standard.operators.python.BranchPythonVirtualenvOperator +airflow.providers.standard.operators.python.ExternalPythonOperator +airflow.providers.standard.operators.python.PythonOperator +airflow.providers.standard.operators.python.PythonVirtualenvOperator +airflow.providers.standard.operators.python.ShortCircuitOperator +airflow.providers.standard.operators.smooth.SmoothOperator +airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunOperator +airflow.providers.standard.operators.weekday.BranchDayOfWeekOperator +airflow.providers.standard.sensors.bash.BashSensor +airflow.providers.standard.sensors.date_time.DateTimeSensor +airflow.providers.standard.sensors.date_time.DateTimeSensorAsync +airflow.providers.standard.sensors.external_task.ExternalDagLink +airflow.providers.standard.sensors.external_task.ExternalTaskMarker +airflow.providers.standard.sensors.external_task.ExternalTaskSensor +airflow.providers.standard.sensors.filesystem.FileSensor +airflow.providers.standard.sensors.python.PythonSensor +airflow.providers.standard.sensors.time.TimeSensor +airflow.providers.standard.sensors.time.TimeSensorAsync +airflow.providers.standard.sensors.time_delta.TimeDeltaSensor +airflow.providers.standard.sensors.time_delta.TimeDeltaSensorAsync +airflow.providers.standard.sensors.weekday.DayOfWeekSensor +airflow.providers.standard.triggers.external_task.DagStateTrigger +airflow.providers.standard.triggers.external_task.WorkflowTrigger +airflow.providers.standard.triggers.file.FileTrigger +airflow.providers.standard.triggers.temporal.DateTimeTrigger +airflow.providers.standard.triggers.temporal.TimeDeltaTrigger diff --git a/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py b/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py index 4a4fabd320f..8baf8397611 100644 --- a/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py +++ b/scripts/tests/ci/prek/test_check_core_shim_targets_exist.py @@ -19,7 +19,13 @@ from __future__ import annotations import textwrap import pytest -from check_core_shim_targets_exist import find_provider_targets, is_target_present, main +from check_core_shim_targets_exist import ( + find_provider_targets, + is_entry_present, + is_target_present, + main, + read_snapshot, +) MODULE = "airflow.providers.acme.sensors.wait" @@ -93,5 +99,44 @@ def test_is_target_present(providers_root, key, target, expected): assert is_target_present(key, target, providers_root) is expected +def test_read_snapshot_skips_comments_and_blank_lines(write_text_file): + path = write_text_file(f"# header\n\n{MODULE}.WaitSensor\n{MODULE}.*\n") + + assert read_snapshot(path) == {f"{MODULE}.WaitSensor": 3, f"{MODULE}.*": 4} + + [email protected]( + ("entry", "expected"), + [ + pytest.param(f"{MODULE}.WaitSensor", True, id="class"), + pytest.param(f"{MODULE}.*", True, id="wildcard"), + pytest.param("airflow.providers.acme.sensors.gone.*", False, id="removed-wildcard-module"), + pytest.param(f"{MODULE}.WaitSensorAsync", False, id="removed-class"), + ], +) +def test_is_entry_present(providers_root, entry, expected): + assert is_entry_present(entry, providers_root) is expected + + [email protected]( + ("shim_targets", "snapshot_targets", "expected"), + [ + pytest.param(["WaitSensor"], ["WaitSensor"], 0, id="recorded-and-present"), + pytest.param(["WaitSensor"], [], 1, id="shim-target-not-recorded"), + pytest.param(["WaitSensorAsync"], [], 1, id="unrecorded-shim-target-missing"), + pytest.param([], ["WaitSensorAsync"], 1, id="shim-and-target-removed-together"), + ], +) +def test_main(tmp_path, providers_root, shim_targets, snapshot_targets, expected): + core_src = tmp_path / "core" + core_src.mkdir() + entries = ", ".join(f'"{name}": "{MODULE}.{name}"' for name in shim_targets) + (core_src / "shims.py").write_text(f"add_deprecated_classes({{'wait': {{{entries}}}}}, __name__)\n") + snapshot = tmp_path / "snapshot.txt" + snapshot.write_text("".join(f"{MODULE}.{name}\n" for name in snapshot_targets)) + + assert main(core_src, providers_root, snapshot) == expected + + def test_main_passes_on_repository(): assert main() == 0
