This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b5e9bae6dc3 Fix wave-release auto-trigger meta-token expansion (#66100)
b5e9bae6dc3 is described below
commit b5e9bae6dc34d7b7cd1f63a1069439d414635b62
Author: Kaxil Naik <[email protected]>
AuthorDate: Thu Apr 30 01:17:50 2026 +0100
Fix wave-release auto-trigger meta-token expansion (#66100)
`publish-docs-to-s3.yml`'s build-info job dropped `all-providers` and
`apache-airflow-providers` meta-tokens when computing `registry-providers`,
which left the `update-registry` gate empty and silently skipped the
registry update on every wave release.
This fix extracts the registry-trigger logic into a small, unit-tested
Python script (`dev/registry/derive_wave_providers.py`). When `INCLUDE_DOCS`
contains a meta-token (`all`, `all-providers`, `apache-airflow-providers`)
AND the dispatch ref matches the wave-tag pattern (`providers/YYYY-MM-DD`),
the script derives the wave's actual provider list from per-provider tags
reachable from the wave ref but not from the previous wave tag. Only those
providers go through the incremental registry build; non-wave providers'
download counts and latest pointers stay untouched.
For non-wave-tag dispatches (e.g., manual rebuild against `main`) or when
the derivation produces an empty list, the script falls back to a full
registry rebuild via a new `registry-full-build` output, which the gate
honors alongside the existing per-provider list. The downstream
`registry-build.yml` already interprets `provider: ""` as "full build", so
no change is needed there.
---
.github/workflows/publish-docs-to-s3.yml | 39 +++--
dev/registry/derive_wave_providers.py | 173 +++++++++++++++++++++++
dev/registry/tests/test_derive_wave_providers.py | 169 ++++++++++++++++++++++
uv.lock | 2 +
4 files changed, 360 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index 56273d8db9a..53ff431a8ec 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -97,7 +97,8 @@ jobs:
# yamllint disable rule:line-length
skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder &&
'--skip-write-to-stable-folder' || '' }}
default-python-version: "3.10"
- registry-providers: ${{ steps.parameters.outputs.registry-providers }}
+ registry-providers: ${{
steps.derive_registry_inputs.outputs.registry-providers }}
+ registry-full-build: ${{
steps.derive_registry_inputs.outputs.registry-full-build }}
if: contains(fromJSON('[
"ashb",
"bugraoz93",
@@ -114,6 +115,19 @@ jobs:
"vatsrahul1001",
]'), github.event.sender.login)
steps:
+ - name: "Checkout for wave provider derivation"
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #
v6.0.2
+ with:
+ persist-credentials: false
+ ref: ${{ inputs.ref }}
+ fetch-tags: true
+ fetch-depth: 0
+ - name: "Derive registry trigger inputs"
+ id: derive_registry_inputs
+ env:
+ INCLUDE_DOCS: ${{ inputs.include-docs }}
+ REF: ${{ inputs.ref }}
+ run: python3 dev/registry/derive_wave_providers.py
- name: "Input parameters summary"
shell: bash
id: parameters
@@ -165,27 +179,6 @@ jobs:
echo "airflow-version=no-airflow" >> ${GITHUB_OUTPUT}
echo "airflow-base-version=no-airflow" >> ${GITHUB_OUTPUT}
fi
- # Extract provider IDs for registry update
- # include-docs contains short names like "amazon", "common.ai",
"openlineage"
- # or full package names like "apache-airflow-providers-amazon"
- REGISTRY_PROVIDERS=""
- for pkg in ${INCLUDE_DOCS}; do
- case "${pkg}" in
- apache-airflow-providers-*)
- PROVIDER_ID="${pkg#apache-airflow-providers-}"
-
REGISTRY_PROVIDERS="${REGISTRY_PROVIDERS:+${REGISTRY_PROVIDERS} }${PROVIDER_ID}"
- ;;
-
apache-airflow|all-providers|apache-airflow-providers|helm-chart|docker-stack|apache-airflow-ctl|apache-airflow-task-sdk)
- # Not a provider package, skip
- ;;
- *)
- # Short name like "amazon" or "common.ai" -- convert dots to
hyphens
- PROVIDER_ID="${pkg//./-}"
-
REGISTRY_PROVIDERS="${REGISTRY_PROVIDERS:+${REGISTRY_PROVIDERS} }${PROVIDER_ID}"
- ;;
- esac
- done
- echo "registry-providers=${REGISTRY_PROVIDERS}" >> ${GITHUB_OUTPUT}
build-docs:
needs: [build-info]
@@ -434,7 +427,7 @@ jobs:
update-registry:
needs: [publish-docs-to-s3, build-info]
- if: needs.build-info.outputs.registry-providers != ''
+ if: needs.build-info.outputs.registry-providers != '' ||
needs.build-info.outputs.registry-full-build == 'true'
name: "Update Provider Registry"
permissions:
contents: read
diff --git a/dev/registry/derive_wave_providers.py
b/dev/registry/derive_wave_providers.py
new file mode 100644
index 00000000000..7a473e8a9eb
--- /dev/null
+++ b/dev/registry/derive_wave_providers.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Derive registry trigger inputs for publish-docs-to-s3.yml.
+
+Implements the gating logic for astronomer/issues-airflow#1305.
+
+Wave dispatches send INCLUDE_DOCS="all-providers apache-airflow-providers"
+(see dev/breeze/src/airflow_breeze/commands/workflow_commands.py:188-195
+and dev/README_RELEASE_PROVIDERS.md). When a meta-token is present and the
+ref is a wave tag (providers/YYYY-MM-DD), this script derives the wave's
+actual provider list from per-provider tags reachable from the wave ref
+but not from the previous wave tag, so registry-build runs incrementally
+on just those providers. Falls back to a full rebuild for non-wave refs
+or when the derivation produces an empty list.
+
+Tag-push ordering: README_RELEASE_PROVIDERS.md flows the wave-tag push
+BEFORE provider RC tags. Today publish-docs-to-s3 is dispatched manually
+AFTER all provider tags exist, so the derivation always has its inputs.
+Any future auto-trigger on wave-tag push must address this race.
+
+Inputs (env): INCLUDE_DOCS, REF, GITHUB_OUTPUT.
+Outputs (written to GITHUB_OUTPUT):
+ registry-providers space-separated provider IDs, or "" for full build
+ registry-full-build "true" for full rebuild, "false" otherwise
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+
+WAVE_TAG_RE = re.compile(r"^providers/[0-9]{4}-[0-9]{2}-[0-9]{2}$")
+WAVE_TAG_GLOB = "providers/[0-9]*-*-*"
+PROVIDER_TAG_GLOB = "providers-*/*"
+PROVIDER_TAG_RE = re.compile(r"^providers-(?P<id>.+?)/(?P<ver>.+)$")
+RC_SUFFIX_RE = re.compile(r"rc[0-9]+$")
+
+META_TOKENS = frozenset({"all", "all-providers", "apache-airflow-providers"})
+NON_PROVIDER_TOKENS = frozenset(
+ {
+ "apache-airflow",
+ "helm-chart",
+ "docker-stack",
+ "apache-airflow-ctl",
+ "apache-airflow-task-sdk",
+ }
+)
+PROVIDER_PREFIX = "apache-airflow-providers-"
+
+
+def _git(*args: str) -> list[str]:
+ result = subprocess.run(["git", *args], capture_output=True, text=True,
check=True)
+ return [line for line in result.stdout.splitlines() if line]
+
+
+def previous_wave_tag(ref: str, wave_tags: list[str]) -> str | None:
+ """Return the wave tag immediately preceding ref, or None.
+
+ Filters input through WAVE_TAG_RE because the git glob
`providers/[0-9]*-*-*`
+ is broader than the strict `providers/YYYY-MM-DD` shape and matches
+ unrelated tags (e.g. `providers/2.11/2026-02-16` for an Airflow 2.11
+ release with a date suffix).
+ """
+ valid = [t for t in wave_tags if WAVE_TAG_RE.match(t)]
+ try:
+ idx = valid.index(ref)
+ except ValueError:
+ return None
+ return valid[idx + 1] if idx + 1 < len(valid) else None
+
+
+def wave_provider_ids(tags: list[str]) -> list[str]:
+ """Provider IDs from per-provider tags, dropping rc tags. Sorted."""
+ ids: set[str] = set()
+ for tag in tags:
+ if RC_SUFFIX_RE.search(tag):
+ continue
+ match = PROVIDER_TAG_RE.match(tag)
+ if match:
+ ids.add(match.group("id"))
+ return sorted(ids)
+
+
+def parse_explicit_packages(include_docs: str) -> list[str]:
+ """Per-package incremental path -- INCLUDE_DOCS without meta-tokens."""
+ providers: list[str] = []
+ seen: set[str] = set()
+ for token in include_docs.split():
+ if token in NON_PROVIDER_TOKENS or token in META_TOKENS:
+ continue
+ if token.startswith(PROVIDER_PREFIX):
+ provider_id = token[len(PROVIDER_PREFIX) :]
+ else:
+ provider_id = token.replace(".", "-")
+ if provider_id and provider_id not in seen:
+ seen.add(provider_id)
+ providers.append(provider_id)
+ return providers
+
+
+def derive(include_docs: str, ref: str, *, git_runner=_git) -> tuple[str,
bool, str]:
+ """Return (registry_providers, full_build, log_message).
+
+ Pure function -- pass git_runner=fake for tests.
+ """
+ tokens = include_docs.split()
+ has_meta = any(t in META_TOKENS for t in tokens)
+
+ if not has_meta:
+ providers = parse_explicit_packages(include_docs)
+ return " ".join(providers), False, ""
+
+ if not WAVE_TAG_RE.match(ref):
+ return "", True, f"REF={ref} is not a wave-tag pattern; using full
rebuild for meta-token dispatch."
+
+ wave_tags = git_runner("tag", "--list", WAVE_TAG_GLOB,
"--sort=-creatordate")
+ prev = previous_wave_tag(ref, wave_tags)
+ if not prev:
+ return (
+ "",
+ True,
+ f"::warning::No predecessor wave tag found before {ref}; falling
back to full rebuild.",
+ )
+
+ raw_tags = git_runner("tag", "--merged", ref, "--no-merged", prev,
"--list", PROVIDER_TAG_GLOB)
+ providers = wave_provider_ids(raw_tags)
+ if not providers:
+ return (
+ "",
+ True,
+ f"::warning::Wave ref {ref} has no new per-provider tags since
{prev}; falling back to full rebuild.",
+ )
+
+ joined = " ".join(providers)
+ return joined, False, f"Derived wave providers ({prev} -> {ref}): {joined}"
+
+
+def main() -> int:
+ include_docs = os.environ.get("INCLUDE_DOCS", "")
+ ref = os.environ.get("REF", "")
+ output_path = os.environ.get("GITHUB_OUTPUT")
+ if not output_path:
+ print("GITHUB_OUTPUT not set", file=sys.stderr)
+ return 1
+
+ registry_providers, full_build, log = derive(include_docs, ref)
+ if log:
+ print(log)
+ with open(output_path, "a", encoding="utf-8") as fh:
+ fh.write(f"registry-providers={registry_providers}\n")
+ fh.write(f"registry-full-build={'true' if full_build else 'false'}\n")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/dev/registry/tests/test_derive_wave_providers.py
b/dev/registry/tests/test_derive_wave_providers.py
new file mode 100644
index 00000000000..32a35d93b95
--- /dev/null
+++ b/dev/registry/tests/test_derive_wave_providers.py
@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for dev/registry/derive_wave_providers.py."""
+
+from __future__ import annotations
+
+import pytest
+from derive_wave_providers import derive
+
+
+def fake_git(wave_tags=None, provider_tags=None):
+ """Build a git_runner that returns canned outputs based on the args."""
+ wave_tags = wave_tags or []
+ provider_tags = provider_tags or []
+
+ def runner(*args):
+ if args[:2] == ("tag", "--list") and "providers/[0-9]*-*-*" in args:
+ return list(wave_tags)
+ if args[:2] == ("tag", "--merged"):
+ return list(provider_tags)
+ raise AssertionError(f"unexpected git invocation: {args}")
+
+ return runner
+
+
+WAVE_TAGS = [
+ "providers/2026-04-21",
+ "providers/2026-04-12",
+ "providers/2026-03-17",
+]
+
+WAVE_PROVIDER_TAGS = [
+ "providers-amazon/9.26.0",
+ "providers-amazon/9.26.0rc1",
+ "providers-google/21.2.0",
+ "providers-google/21.2.0rc1",
+ "providers-vespa/0.1.0",
+ "providers-microsoft-azure/13.1.2",
+ "providers-microsoft-azure/13.1.2rc1",
+]
+
+
[email protected](
+ ("include_docs", "ref", "expected_providers", "expected_full_build",
"log_substr"),
+ [
+ # Standard wave dispatch: derives from git tags.
+ (
+ "all-providers apache-airflow-providers",
+ "providers/2026-04-21",
+ "amazon google microsoft-azure vespa",
+ False,
+ "Derived wave providers (providers/2026-04-12 ->
providers/2026-04-21)",
+ ),
+ # Meta-token but ref is RC-suffixed (not a wave tag).
+ (
+ "all-providers apache-airflow-providers",
+ "providers/2026-04-21-rc1",
+ "",
+ True,
+ "not a wave-tag pattern",
+ ),
+ # Meta-token but ref is main.
+ ("all-providers", "main", "", True, "not a wave-tag pattern"),
+ # Meta-token, ref is sha.
+ ("apache-airflow-providers", "f8a258069f", "", True, "not a wave-tag
pattern"),
+ # Explicit packages -- incremental path.
+ ("amazon google", "providers/2026-04-21", "amazon google", False, ""),
+ # Explicit with full-package-name token.
+ (
+ "apache-airflow-providers-amazon",
+ "providers-amazon/9.27.0",
+ "amazon",
+ False,
+ "",
+ ),
+ # Explicit with dot-to-hyphen conversion.
+ ("common.ai", "any", "common-ai", False, ""),
+ # Explicit with non-provider tokens stripped.
+ ("apache-airflow helm-chart docker-stack", "any", "", False, ""),
+ # Explicit dedup.
+ ("amazon amazon google", "any", "amazon google", False, ""),
+ # Empty.
+ ("", "any", "", False, ""),
+ ],
+)
+def test_derive(include_docs, ref, expected_providers, expected_full_build,
log_substr):
+ git = fake_git(wave_tags=WAVE_TAGS, provider_tags=WAVE_PROVIDER_TAGS)
+ providers, full_build, log = derive(include_docs, ref, git_runner=git)
+ assert providers == expected_providers
+ assert full_build is expected_full_build
+ if log_substr:
+ assert log_substr in log
+
+
+def test_first_wave_no_predecessor():
+ # Ref IS a wave tag but there's no predecessor in the list.
+ git = fake_git(wave_tags=["providers/2026-01-15"], provider_tags=[])
+ providers, full_build, log = derive("all-providers",
"providers/2026-01-15", git_runner=git)
+ assert providers == ""
+ assert full_build is True
+ assert "::warning::No predecessor wave tag found" in log
+
+
+def test_wave_with_no_new_provider_tags():
+ # Ref is a known wave tag with predecessor, but no per-provider tags
+ # between them (suspect: race with tag push, or no-op republish).
+ git = fake_git(
+ wave_tags=["providers/2026-05-19", "providers/2026-04-21"],
+ provider_tags=[],
+ )
+ providers, full_build, log = derive("all-providers",
"providers/2026-05-19", git_runner=git)
+ assert providers == ""
+ assert full_build is True
+ assert "::warning::Wave ref providers/2026-05-19 has no new per-provider
tags" in log
+
+
+def test_only_rc_tags_between_waves():
+ # All per-provider tags between the two waves are RCs -- treated as no
+ # new finals, falls back to full rebuild with warning.
+ git = fake_git(
+ wave_tags=["providers/2026-05-19", "providers/2026-04-21"],
+ provider_tags=["providers-amazon/9.27.0rc1",
"providers-google/21.3.0rc1"],
+ )
+ providers, full_build, log = derive("all-providers",
"providers/2026-05-19", git_runner=git)
+ assert providers == ""
+ assert full_build is True
+ assert "::warning::" in log
+
+
+def test_meta_token_with_explicit_provider():
+ # Meta-token wins -- explicit provider tokens are dropped.
+ git = fake_git(wave_tags=WAVE_TAGS, provider_tags=WAVE_PROVIDER_TAGS)
+ providers, full_build, log = derive("amazon all-providers",
"providers/2026-04-21", git_runner=git)
+ assert providers == "amazon google microsoft-azure vespa"
+ assert full_build is False
+
+
+def test_glob_matches_non_wave_tag_is_filtered_out():
+ # The git glob `providers/[0-9]*-*-*` is broader than the strict
+ # `providers/YYYY-MM-DD` regex and matches release-with-date tags like
+ # `providers/2.11/2026-02-16` (Airflow 2.11 release on that date).
+ # Predecessor lookup must skip those so it picks the actual prior wave.
+ git = fake_git(
+ wave_tags=[
+ "providers/2026-02-20",
+ "providers/2.11/2026-02-16", # noise: not a wave tag
+ "providers/2026-02-10",
+ ],
+ provider_tags=["providers-amazon/9.20.0"],
+ )
+ providers, full_build, log = derive("all-providers",
"providers/2026-02-20", git_runner=git)
+ # Predecessor must be the real prior wave (2026-02-10), not the noise.
+ assert providers == "amazon"
+ assert full_build is False
+ assert "providers/2026-02-10 -> providers/2026-02-20" in log
diff --git a/uv.lock b/uv.lock
index 935bae223c8..d72bc255161 100644
--- a/uv.lock
+++ b/uv.lock
@@ -7911,6 +7911,7 @@ name = "apache-airflow-registry-tools"
version = "0.0.1"
source = { editable = "dev/registry" }
dependencies = [
+ { name = "packaging" },
{ name = "pydantic" },
{ name = "pyyaml" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
@@ -7923,6 +7924,7 @@ dev = [
[package.metadata]
requires-dist = [
+ { name = "packaging", specifier = ">=24.0" },
{ name = "pydantic", specifier = ">=2.12.0" },
{ name = "pyyaml", specifier = ">=6.0.3" },
{ name = "tomli", marker = "python_full_version < '3.11'", specifier =
">=2.0.0" },
