This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ece27bba4a5 Regenerate stale provider dependencies instead of silently
using them (#70955)
ece27bba4a5 is described below
commit ece27bba4a5cffeb96aa6a8aad5faf3e63f7288c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Aug 3 09:27:57 2026 +0200
Regenerate stale provider dependencies instead of silently using them
(#70955)
Release commands enumerate providers from
generated/provider_dependencies.json,
filtering on each provider's state. The freshness check runs on every breeze
invocation, but when the json was present and its sha256sum sidecar was
not, it
delegated to a helper that only regenerates when the json is missing. The
stale
file was therefore returned as-is and no sidecar was ever written, leaving
the
check permanently disabled for that checkout.
A provider whose state changed then stays invisible to the release:
common.ai
was flipped to ready in 682cf06525 but a checkout carrying an older json
still
called it not-ready, so it was dropped from the 2026-08-01 wave with no
warning
and was only caught by an unrelated package count mismatch.
---
.../airflow_breeze/utils/provider_dependencies.py | 20 ++++--
dev/breeze/tests/test_provider_dependencies.py | 81 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 7 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
index d67e0ec9bfc..02aa337de37 100644
--- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
@@ -179,24 +179,30 @@ def _calculate_provider_deps_hash():
def get_provider_dependencies() -> dict:
if not PROVIDER_DEPENDENCIES_JSON_PATH.exists():
calculated_hash = _calculate_provider_deps_hash()
- PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n")
# We use regular print there as rich console might not be initialized
yet here
print("Regenerating provider dependencies file")
regenerate_provider_dependencies_once()
+ # Only record the hash once regeneration succeeded, otherwise a failed
run would
+ # leave a sidecar claiming that a missing/stale file is up to date.
+ PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n")
return json.loads(PROVIDER_DEPENDENCIES_JSON_PATH.read_text())
+def _force_regenerate_provider_dependencies() -> None:
+ # get_provider_dependencies() only regenerates when the JSON is absent, so
the file has
+ # to be removed for it to pick up changed provider.yaml/pyproject.toml
contents.
+ PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True)
+ get_provider_dependencies.cache_clear()
+ get_provider_dependencies()
+
+
def generate_provider_dependencies_if_needed():
if not PROVIDER_DEPENDENCIES_JSON_PATH.exists() or not
PROVIDER_DEPENDENCIES_JSON_HASH_PATH.exists():
- get_provider_dependencies.cache_clear()
- get_provider_dependencies()
+ _force_regenerate_provider_dependencies()
else:
calculated_hash = _calculate_provider_deps_hash()
if calculated_hash.strip() !=
PROVIDER_DEPENDENCIES_JSON_HASH_PATH.read_text().strip():
- # Force re-generation
- PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True)
- get_provider_dependencies.cache_clear()
- get_provider_dependencies()
+ _force_regenerate_provider_dependencies()
def get_related_providers(
diff --git a/dev/breeze/tests/test_provider_dependencies.py
b/dev/breeze/tests/test_provider_dependencies.py
index 4ddc16a60cf..bf77f6dd29d 100644
--- a/dev/breeze/tests/test_provider_dependencies.py
+++ b/dev/breeze/tests/test_provider_dependencies.py
@@ -16,9 +16,13 @@
# under the License.
from __future__ import annotations
+from unittest import mock
+
import pytest
+from airflow_breeze.utils import provider_dependencies as
provider_dependencies_module
from airflow_breeze.utils.provider_dependencies import (
+ generate_provider_dependencies_if_needed,
get_related_providers,
)
@@ -47,3 +51,80 @@ def test_both():
def test_none():
with pytest.raises(ValueError, match=r".*must be.*"):
get_related_providers("trino", upstream_dependencies=False,
downstream_dependencies=False)
+
+
[email protected]
+def provider_deps_files(tmp_path):
+ """Point the module at a throwaway json/sha256sum pair and clear the
lru_cache around it."""
+ json_path = tmp_path / "provider_dependencies.json"
+ hash_path = tmp_path / "provider_dependencies.json.sha256sum"
+ provider_dependencies_module.get_provider_dependencies.cache_clear()
+ with mock.patch.multiple(
+ provider_dependencies_module,
+ PROVIDER_DEPENDENCIES_JSON_PATH=json_path,
+ PROVIDER_DEPENDENCIES_JSON_HASH_PATH=hash_path,
+ ):
+ yield json_path, hash_path
+ provider_dependencies_module.get_provider_dependencies.cache_clear()
+
+
[email protected](
+ "hash_sidecar_present",
+ [
+ pytest.param(False, id="missing-sidecar"),
+ pytest.param(True, id="stale-sidecar"),
+ ],
+)
[email protected](provider_dependencies_module,
"regenerate_provider_dependencies_once")
[email protected](provider_dependencies_module,
"_calculate_provider_deps_hash")
+def test_stale_dependencies_are_regenerated(
+ mock_hash, mock_regenerate, provider_deps_files, hash_sidecar_present
+):
+ """A present-but-outdated json must be regenerated, and the sidecar
refreshed.
+
+ Without this, breeze silently enumerates providers from stale state -
which dropped
+ common.ai from the 2026-08-01 provider release wave.
+ """
+ json_path, hash_path = provider_deps_files
+ json_path.write_text('{"common.ai": {"state": "not-ready"}}')
+ if hash_sidecar_present:
+ hash_path.write_text("stale-hash\n")
+ mock_hash.return_value = "fresh-hash"
+
+ def regenerate():
+ json_path.write_text('{"common.ai": {"state": "ready"}}')
+
+ mock_regenerate.side_effect = regenerate
+
+ generate_provider_dependencies_if_needed()
+
+ mock_regenerate.assert_called_once()
+ assert provider_dependencies_module.get_provider_dependencies() ==
{"common.ai": {"state": "ready"}}
+ assert hash_path.read_text().strip() == "fresh-hash"
+
+
[email protected](provider_dependencies_module,
"regenerate_provider_dependencies_once")
[email protected](provider_dependencies_module,
"_calculate_provider_deps_hash")
+def test_up_to_date_dependencies_are_not_regenerated(mock_hash,
mock_regenerate, provider_deps_files):
+ json_path, hash_path = provider_deps_files
+ json_path.write_text('{"common.ai": {"state": "ready"}}')
+ hash_path.write_text("fresh-hash\n")
+ mock_hash.return_value = "fresh-hash"
+
+ generate_provider_dependencies_if_needed()
+
+ mock_regenerate.assert_not_called()
+
+
[email protected](provider_dependencies_module,
"regenerate_provider_dependencies_once")
[email protected](provider_dependencies_module,
"_calculate_provider_deps_hash")
+def test_hash_not_written_when_regeneration_fails(mock_hash, mock_regenerate,
provider_deps_files):
+ """A failed regeneration must not leave a sidecar vouching for stale
content."""
+ _, hash_path = provider_deps_files
+ mock_hash.return_value = "fresh-hash"
+ mock_regenerate.side_effect = RuntimeError("boom")
+
+ with pytest.raises(RuntimeError, match="boom"):
+ generate_provider_dependencies_if_needed()
+
+ assert not hash_path.exists()
