This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new 577ef832825 [v3-2-test] breeze: prune unreleased versions from
provider.yaml during metadata generation (#65986) (#66005)
577ef832825 is described below
commit 577ef832825d0a3d93a5c779fe129d0b32f0569c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri May 1 17:22:02 2026 +0200
[v3-2-test] breeze: prune unreleased versions from provider.yaml during
metadata generation (#65986) (#66005)
Adds a hygiene pre-pass to ``breeze release-management
generate-providers-metadata``:
for each provider, every entry in ``provider.yaml``'s ``versions:`` list
except the
first is checked against PyPI. Entries PyPI doesn't recognise are stale
(e.g. a
release that was prepared but never published) and get removed from the
file in
place, so generated metadata reflects only versions a user can actually
install.
The first entry is intentionally skipped — it is the in-progress next
release and
is allowed to predate publication. PyPI lookups run in parallel; on network
failure the file is left untouched. The version-line removal is regex-based
on
the exact shape produced by ``_update_version_in_provider_yaml`` in
``provider_documentation.py``, so comments and surrounding entries are
preserved.
(cherry picked from commit b28a45e7b5eb33126f112dfc58d3721881364427)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../commands/release_management_commands.py | 33 +++++++++-
.../airflow_breeze/utils/provider_dependencies.py | 73 ++++++++++++++++++++++
2 files changed, 105 insertions(+), 1 deletion(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index fe663a2925e..1cde739ce2a 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -3102,6 +3102,38 @@ def generate_providers_metadata(
console_print(metadata_dict)
return
+ package_ids = list(get_provider_dependencies().keys())
+
+ # Hygiene pass: every entry in `provider.yaml`'s `versions:` list except
+ # the first is supposed to be a published PyPI release. The first entry
+ # is the in-progress next release and is allowed to predate publication;
+ # the rest must be reachable on PyPI. Any older entry that PyPI does not
+ # know about is stale (e.g. a release that was prepared but never
+ # published) — drop it from `provider.yaml` so the metadata generated
+ # below reflects only versions a user can actually install.
+ from airflow_breeze.utils.packages import
get_provider_distributions_metadata
+ from airflow_breeze.utils.provider_dependencies import (
+ prune_unreleased_versions_from_provider_yaml,
+ )
+
+ console_print("\n[info]Checking provider.yaml versions[1:] against PyPI
for stale entries...[/]\n")
+ with Pool() as pypi_pool:
+ pruned_per_provider =
pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids)
+ total_pruned = 0
+ for pid, pruned in zip(package_ids, pruned_per_provider):
+ if pruned:
+ console_print(f"[warning]{pid}: removed unreleased versions from
provider.yaml: {pruned}[/]")
+ total_pruned += len(pruned)
+ if total_pruned:
+ console_print(
+ f"\n[warning]Removed {total_pruned} unreleased version entr"
+ f"{'y' if total_pruned == 1 else 'ies'} from provider.yaml files. "
+ "Re-reading provider metadata.[/]\n"
+ )
+ get_provider_distributions_metadata.cache_clear()
+ else:
+ console_print("[info]All provider.yaml versions[1:] are present on
PyPI.[/]\n")
+
partial_generate_providers_metadata = partial(
generate_providers_metadata_for_provider,
provider_version=None,
@@ -3110,7 +3142,6 @@ def generate_providers_metadata(
airflow_release_dates=airflow_release_dates,
current_metadata=current_metadata,
)
- package_ids = get_provider_dependencies().keys()
with Pool() as pool:
results = pool.map(
partial_generate_providers_metadata,
diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
index ecfaaf2b89c..38e19cbf32d 100644
--- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py
@@ -22,6 +22,8 @@ import re
import shutil
import subprocess
import sys
+import urllib.error
+import urllib.request
from collections.abc import Generator
from functools import cache, partial
from multiprocessing import Pool
@@ -88,6 +90,77 @@ def regenerate_provider_dependencies_once() -> None:
)
+def _fetch_pypi_released_versions(package_name: str) -> set[str] | None:
+ """
+ Fetch the set of versions of ``package_name`` released on PyPI.
+
+ Returns ``None`` on network/HTTP error so the caller can skip pruning
+ rather than crash a long batch run.
+ """
+ pypi_url = f"https://pypi.org/pypi/{package_name}/json";
+ try:
+ with urllib.request.urlopen(pypi_url, timeout=30) as resp:
+ data = json.loads(resp.read().decode("utf-8"))
+ except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as e:
+ console_print(f"[warning]Could not fetch PyPI data for {package_name}:
{e}")
+ return None
+ return set((data.get("releases") or {}).keys())
+
+
+def prune_unreleased_versions_from_provider_yaml(provider_id: str) ->
list[str]:
+ """
+ Check ``versions[1:]`` in a provider's ``provider.yaml`` against PyPI.
+
+ The first entry is intentionally skipped — it is the in-progress next
+ release, which is allowed to predate its PyPI publication. For every
+ other entry, if PyPI does not list that exact version, the entry is
+ removed from the file in place.
+
+ Returns the list of versions that were removed, in original order.
+ Returns ``[]`` if nothing was pruned (or PyPI lookup failed).
+ """
+ import yaml
+
+ # Local import to avoid a circular import: packages.py already depends on
+ # provider_dependencies.py via get_provider_dependencies() lookups.
+ from airflow_breeze.utils.packages import get_provider_yaml
+
+ provider_yaml_path = get_provider_yaml(provider_id)
+ provider_yaml_text = provider_yaml_path.read_text()
+ provider_yaml_dict = yaml.safe_load(provider_yaml_text)
+ versions: list[str] = provider_yaml_dict.get("versions") or []
+ if len(versions) <= 1:
+ return []
+
+ package_name = "apache-airflow-providers-" + provider_id.replace(".", "-")
+ pypi_released = _fetch_pypi_released_versions(package_name)
+ if pypi_released is None:
+ # On network failure, leave the file alone — better than deleting
+ # entries based on incomplete data.
+ return []
+
+ to_prune = [v for v in versions[1:] if v not in pypi_released]
+ if not to_prune:
+ return []
+
+ new_text = provider_yaml_text
+ for v in to_prune:
+ # Match the exact line shape produced by
_update_version_in_provider_yaml
+ # in provider_documentation.py: two-space indent, "- ", version, EOL.
+ # `[ \t]*\n` (not `\s*\n`) avoids consuming the next line's newline.
+ new_text = re.sub(
+ rf"^[ \t]+-[ \t]+{re.escape(v)}[ \t]*\n",
+ "",
+ new_text,
+ count=1,
+ flags=re.MULTILINE,
+ )
+
+ if new_text != provider_yaml_text:
+ provider_yaml_path.write_text(new_text)
+ return to_prune
+
+
def _calculate_provider_deps_hash():
import hashlib
