This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 271604fa772 Allow manual triggering of constraints refresh workflow
(#69457)
271604fa772 is described below
commit 271604fa772551eee6fa67f834908fddff325cbc
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Jul 6 13:14:41 2026 +0200
Allow manual triggering of constraints refresh workflow (#69457)
Constraints in the constraints-* branches only refresh automatically when
uv.lock changes on a stable branch. After a providers release there is no
such
push, so refreshing constraints - for example to pick up newly released
providers before promoting an RC - required running the breeze
constraint-generation commands locally.
Add a workflow_dispatch trigger to the update-constraints workflow. The
manual
run is launched from main and takes a `ref` input (branch, tag or commit
hash)
selecting the sources to refresh constraints from; the target
constraints-X-Y
branch is derived from that ref's branch_defaults.py. This avoids
cherry-picking
the workflow to vX-Y-test / vX-Y-stable, which matters because those
branches
diverge while RCs are being voted on. A toggle re-resolves to the newest
matching dependencies from PyPI (default on). The ref is threaded through
the
reusable ci-image-build and generate-constraints workflows via a new
optional
checkout-ref input (backward compatible for existing callers).
Because the RC constraints are frozen when the RC is cut, start-release now
asks
whether to base the final constraints-<version> tag on the latest
constraints-X-Y
branch tip (when refreshed after the last RC) instead of the RC tag.
The manual procedure and release-guide note are documented accordingly.
---
.github/workflows/ci-image-build.yml | 6 ++
.github/workflows/generate-constraints.yml | 8 ++-
.github/workflows/update-constraints-on-push.yml | 76 +++++++++++++++++-----
...UALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md | 49 ++++++++++++++
dev/README_RELEASE_AIRFLOW.md | 10 +++
.../src/airflow_breeze/commands/release_command.py | 21 +++++-
dev/breeze/tests/test_release_command.py | 52 +++++++++++++++
7 files changed, 204 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 7f1fa61fe60..980c9c7a223 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -29,6 +29,11 @@ on: # yamllint disable-line rule:truthy
required: false
default: ""
type: string
+ checkout-ref:
+ description: "Commit-ish to checkout sources from (empty = workflow
ref)."
+ required: false
+ default: ""
+ type: string
pull-request-target:
description: "Whether we are running this from pull-request-target
workflow (true/false)"
required: false
@@ -120,6 +125,7 @@ jobs:
- name: "Checkout target branch"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
with:
+ ref: ${{ inputs.checkout-ref }}
persist-credentials: false
- name: "Free up disk space"
shell: bash
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 55604039f77..2a8d8f4f72d 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -52,6 +52,11 @@ on: # yamllint disable-line rule:truthy
description: "Whether to use uvloop (true/false)"
required: true
type: string
+ checkout-ref:
+ description: "Commit-ish to checkout sources from (empty = workflow
ref)."
+ required: false
+ default: ""
+ type: string
jobs:
generate-constraints-matrix:
permissions:
@@ -76,9 +81,10 @@ jobs:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ - name: "Checkout ${{ inputs.checkout-ref || github.ref }} ( ${{
github.sha }} )"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
with:
+ ref: ${{ inputs.checkout-ref }}
persist-credentials: false
- name: "Install prek"
uses: ./.github/actions/install-prek
diff --git a/.github/workflows/update-constraints-on-push.yml
b/.github/workflows/update-constraints-on-push.yml
index 16460bd89cb..b65168c53da 100644
--- a/.github/workflows/update-constraints-on-push.yml
+++ b/.github/workflows/update-constraints-on-push.yml
@@ -16,7 +16,20 @@
# under the License.
#
---
-name: Update constraints on push for main (only when uv.lock changes)
+name: Update constraints (on uv.lock push or manual dispatch)
+# This workflow refreshes the pinned constraint files stored in the
+# `constraints-*` branches. It runs automatically whenever `uv.lock` changes on
+# `main` or a `vX-Y-test` branch, and can also be triggered manually via the
+# "Run workflow" button (workflow_dispatch) - for example to pick up newly
+# released providers/dependencies from PyPI just before promoting an RC.
+#
+# The manual run is always launched from `main` (so the workflow definition and
+# `breeze` come from `main`), and the `ref` input selects the commit-ish
+# (branch, tag or commit hash) whose sources the constraints are refreshed
from.
+# The `constraints-X-Y` branch to push to is derived from that ref's
+# `dev/breeze/src/airflow_breeze/branch_defaults.py`, so no cherry-pick to the
+# `vX-Y-test` / `vX-Y-stable` branch is needed.
+# See dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md for details.
on: # yamllint disable-line rule:truthy
push:
branches:
@@ -24,11 +37,27 @@ on: # yamllint disable-line rule:truthy
- v[0-9]+-[0-9]+-test
paths:
- 'uv.lock'
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: >-
+ Commit-ish to refresh constraints from (branch, tag or commit hash),
+ e.g. `v3-3-test`, `v3-3-stable`, `constraints-3-3` or a tag/hash.
+ The matching `constraints-X-Y` branch is derived from that ref.
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ description: >-
+ Re-resolve to the newest matching dependencies (picks up newly
+ released providers/dependencies from PyPI). Leave enabled when
+ refreshing constraints before promoting an RC.
+ type: boolean
+ default: true
permissions:
contents: read
concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.ref }}
cancel-in-progress: true
env:
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -39,6 +68,25 @@ jobs:
build-info:
name: "Build info"
runs-on: ["ubuntu-22.04"]
+ # Automatic uv.lock-push runs are unrestricted; manual (workflow_dispatch)
+ # runs are limited to release managers (same allowlist as the prod image
+ # release workflow) because they push to the protected constraints-*
branches.
+ if: >-
+ github.event_name != 'workflow_dispatch' ||
+ contains(fromJSON('[
+ "ashb",
+ "bugraoz93",
+ "eladkal",
+ "ephraimbuddy",
+ "jedcunningham",
+ "jscheffl",
+ "kaxil",
+ "pierrejeambrun",
+ "potiuk",
+ "utkarsharma2",
+ "vincbeck",
+ "vatsrahul1001",
+ ]'), github.event.sender.login)
outputs:
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
@@ -49,14 +97,10 @@ jobs:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
- with:
- persist-credentials: false
- - name: Fetch incoming commit ${{ github.sha }} with its parent
+ - name: "Checkout ${{ inputs.ref || github.ref }} ( ${{ inputs.ref ||
github.sha }} )"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
with:
- ref: ${{ github.sha }}
+ ref: ${{ inputs.ref || github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
@@ -75,7 +119,7 @@ jobs:
id: selective-checks
env:
PR_LABELS: "[]"
- COMMIT_REF: "${{ github.sha }}"
+ COMMIT_REF: "${{ inputs.ref || github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
@@ -96,8 +140,11 @@ jobs:
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
+ checkout-ref: ${{ inputs.ref }}
use-uv: "true"
- upgrade-to-newer-dependencies: "false"
+ upgrade-to-newer-dependencies: >-
+ ${{ github.event_name == 'workflow_dispatch'
+ && inputs.upgrade-to-newer-dependencies && 'true' || 'false' }}
docker-cache: "registry"
disable-airflow-repo-cache: "false"
@@ -113,6 +160,7 @@ jobs:
generate-pypi-constraints: "true"
generate-no-providers-constraints: "true"
debug-resources: "false"
+ checkout-ref: ${{ inputs.ref }}
use-uv: "true"
update-constraints:
@@ -125,6 +173,7 @@ jobs:
packages: read
env:
PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
+ CONSTRAINTS_BRANCH: ${{
needs.build-info.outputs.default-constraints-branch }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -133,14 +182,11 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
with:
persist-credentials: false
- - name: "Set constraints branch name"
- id: constraints-branch
- run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
- - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
+ - name: Checkout ${{ env.CONSTRAINTS_BRANCH }}
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 #
v7.0.0
with:
path: "constraints"
- ref: ${{ steps.constraints-branch.outputs.branch }}
+ ref: ${{ env.CONSTRAINTS_BRANCH }}
persist-credentials: true
fetch-depth: 0
- name: "Download constraints from the generate-constraints job"
diff --git a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md
b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md
index 68104fee02d..4a794333ab6 100644
--- a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md
+++ b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md
@@ -31,6 +31,7 @@
- [What the command does](#what-the-command-does)
- [Manually generating constraint files](#manually-generating-constraint-files)
- [Why we need to generate constraint files
manually](#why-we-need-to-generate-constraint-files-manually)
+ - [How to refresh constraints via the CI workflow
(recommended)](#how-to-refresh-constraints-via-the-ci-workflow-recommended)
- [How to generate constraint files](#how-to-generate-constraint-files)
- [Is it safe to generate constraints
manually?](#is-it-safe-to-generate-constraints-manually)
- [Manually updating already tagged constraint
files](#manually-updating-already-tagged-constraint-files)
@@ -213,6 +214,54 @@ with tests, but we KNOW that the tip of the branch is good
and we want to releas
we want to move the PRs of contributors to start using the new constraints.
This should be done with caution
and you need to be sure what you are doing, but you can always do it manually
if you want.
+## How to refresh constraints via the CI workflow (recommended)
+
+The easiest way to refresh the constraints - for example to pick up newly
released
+providers/dependencies from PyPI just before promoting an RC - is to trigger
the
+[`Update constraints`](../.github/workflows/update-constraints-on-push.yml)
workflow manually
+instead of running the `breeze` commands locally. The workflow runs exactly
the same steps
+that run automatically when `uv.lock` changes, builds the CI images, generates
all constraint
+flavours and commits/pushes them to the matching `constraints-*` branch.
+
+The manual run is always launched from `main` and takes a `ref` input that
selects the
+commit-ish (branch, tag or commit hash) whose sources the constraints are
refreshed from. You
+do **not** select the release branch in GitHub's branch dropdown, and you do
**not** need to
+cherry-pick anything to the `vX-Y-test` / `vX-Y-stable` branch first - the
workflow definition
+and `breeze` come from `main`, while the sources come from the `ref` you pass.
The only
+requirement is that the `breeze` constraint-generation commands work against
that `ref` (they
+are stable across recent branches).
+
+Manual runs are restricted to release managers (the same allowlist as the prod
image release
+workflow); the automatic `uv.lock`-push runs are not restricted.
+
+To run it:
+
+1. Go to the [`Update
constraints`](https://github.com/apache/airflow/actions/workflows/update-constraints-on-push.yml)
+ workflow in the Actions tab.
+2. Click **Run workflow** and keep the branch set to `main` (this is where the
workflow runs
+ from - it is not the branch whose constraints get refreshed).
+3. In the **ref** field, enter the commit-ish to refresh constraints from -
for example
+ `v3-3-test`, `v3-3-stable`, `constraints-3-3`, an RC tag, or a commit hash.
The matching
+ `constraints-X-Y` branch to push to is derived automatically from that ref's
+ `dev/breeze/src/airflow_breeze/branch_defaults.py`, so pointing at anything
on the 3.3 line
+ refreshes `constraints-3-3`.
+4. Keep **Re-resolve to the newest matching dependencies** enabled (the
default) so the run
+ picks up the latest released providers/dependencies from PyPI. Disable it
only if you want
+ to regenerate constraints strictly from that ref's `uv.lock` without
upgrading.
+5. Once the run finishes, verify the new commit on the matching constraints
branch
+ (for example `constraints-3-3` for a 3.3 refresh):
+
+ https://github.com/apache/airflow/commits/constraints-<major>-<minor>/
+
+> [!NOTE]
+> Because `v3-3-test` and `v3-3-stable` can diverge while RCs are being voted
on (fixes are
+> cherry-picked to `v3-3-test`), be deliberate about which `ref` you refresh
from. Refresh from
+> the ref that matches the artifacts you are about to promote - typically
`v3-3-stable` (or the
+> RC tag) for a release, not `v3-3-test`.
+
+If you cannot or do not want to use the workflow, you can still generate the
constraints
+locally with the `breeze` commands below.
+
## How to generate constraint files
```bash
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index f7d8dc7961d..7e6005e75bb 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -1372,6 +1372,16 @@ breeze release-management start-release \
Note: The `--task-sdk-version` parameter is optional. If you are releasing
Airflow without a corresponding Task SDK release, you can omit this parameter.
+Note: When it reaches the constraints step, `start-release` asks whether to
base the final
+`constraints-${VERSION}` tag on the latest `constraints-X-Y` branch tip
instead of the RC
+constraints tag. The RC constraints are frozen when the RC is cut, so if any
providers were
+released (or constraints were otherwise refreshed - see
+[MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md](MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md))
+after the last RC and you want the released constraints to reflect that,
answer **yes** to tag the
+`constraints-X-Y` branch tip. Otherwise (the default) the final tag matches
the RC exactly. If you
+do refresh, run the `Update constraints` workflow from `main` with `ref` set
to the ref you are
+releasing (typically `v3-*-stable`) **before** running `start-release`.
+
4. Make sure to update Airflow version in ``v3-*-test`` branch after
cherry-picking to X.Y.1 in
``airflow/__init__.py``
diff --git a/dev/breeze/src/airflow_breeze/commands/release_command.py
b/dev/breeze/src/airflow_breeze/commands/release_command.py
index 5b183e765d9..1b93f9c28ac 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_command.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_command.py
@@ -343,10 +343,27 @@ def upload_to_pypi(version, task_sdk_version=None):
)
+def get_constraints_branch_for_version(version: str) -> str:
+ major, minor = version.split(".")[:2]
+ return f"constraints-{major}-{minor}"
+
+
def retag_constraints(release_candidate, version):
- if confirm_action(f"Retag constraints for {release_candidate} as
{version}?"):
+ # By default the final ``constraints-<version>`` tag is created from the
RC constraints tag.
+ # If the constraints were refreshed after the last RC (e.g. to pick up
newly released
+ # providers - see dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md)
the tip of the
+ # ``constraints-X-Y`` branch is newer than the RC tag and should be tagged
instead.
+ constraints_branch = get_constraints_branch_for_version(version)
+ source_ref = f"constraints-{release_candidate}"
+ if confirm_action(
+ f"Base the final constraints on the latest '{constraints_branch}'
branch tip instead of the "
+ f"'{source_ref}' tag? Choose yes if you refreshed constraints after
{release_candidate}."
+ ):
+ run_command(["git", "fetch", "origin", constraints_branch], check=True)
+ source_ref = f"origin/{constraints_branch}"
+ if confirm_action(f"Retag constraints from {source_ref} as {version}?"):
run_command(
- ["git", "checkout", f"constraints-{release_candidate}"],
+ ["git", "checkout", source_ref],
check=True,
)
run_command(
diff --git a/dev/breeze/tests/test_release_command.py
b/dev/breeze/tests/test_release_command.py
index 2745a8387eb..1e023e6bf5b 100644
--- a/dev/breeze/tests/test_release_command.py
+++ b/dev/breeze/tests/test_release_command.py
@@ -538,3 +538,55 @@ def
test_remove_old_release_no_task_sdk_version(monkeypatch, release_cmd):
assert "task-sdk" not in " ".join(console_messages).lower()
assert run_command_calls == []
assert chdir_calls == [svn_release_repo, "/original/dir"]
+
+
[email protected](
+ ("version", "expected_branch"),
+ [
+ ("3.3.0", "constraints-3-3"),
+ ("3.0.5", "constraints-3-0"),
+ ("3.12.10", "constraints-3-12"),
+ ],
+)
+def test_get_constraints_branch_for_version(release_cmd, version,
expected_branch):
+ assert release_cmd.get_constraints_branch_for_version(version) ==
expected_branch
+
+
+def test_retag_constraints_from_rc_tag_by_default(monkeypatch, release_cmd):
+ run_command_calls: list[list[str]] = []
+
+ def fake_confirm_action(prompt: str, **_kwargs) -> bool:
+ # Decline basing on the branch tip; accept the retag and the push.
+ return not prompt.startswith("Base the final constraints on the
latest")
+
+ monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action)
+ monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs:
run_command_calls.append(cmd))
+
+ release_cmd.retag_constraints("3.3.0rc2", "3.3.0")
+
+ assert ["git", "checkout", "constraints-3.3.0rc2"] in run_command_calls
+ assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for
Apache Airflow 3.3.0"] in (
+ run_command_calls
+ )
+ assert ["git", "push", "origin", "tag", "constraints-3.3.0"] in
run_command_calls
+ # The branch tip is not fetched when basing on the RC tag.
+ assert not any(cmd[:2] == ["git", "fetch"] for cmd in run_command_calls)
+
+
+def test_retag_constraints_from_branch_tip_when_confirmed(monkeypatch,
release_cmd):
+ run_command_calls: list[list[str]] = []
+
+ def fake_confirm_action(prompt: str, **_kwargs) -> bool:
+ return True
+
+ monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action)
+ monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs:
run_command_calls.append(cmd))
+
+ release_cmd.retag_constraints("3.3.0rc2", "3.3.0")
+
+ assert ["git", "fetch", "origin", "constraints-3-3"] in run_command_calls
+ assert ["git", "checkout", "origin/constraints-3-3"] in run_command_calls
+ assert ["git", "checkout", "constraints-3.3.0rc2"] not in run_command_calls
+ assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for
Apache Airflow 3.3.0"] in (
+ run_command_calls
+ )
