This is an automated email from the ASF dual-hosted git repository. vatsrahul1001 pushed a commit to branch beta-aware-start-rc-process in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 73733683965fa20d435896af0eba7fd98d82e4a2 Author: Rahul Vats <[email protected]> AuthorDate: Thu Oct 8 19:49:07 2026 +0530 Support cutting beta pre-releases with start-rc-process Betas and alphas are cut before a vX-Y-stable branch exists, but the command required it for validation, tagging and constraints, so betas had to be cut by hand. Route pre-releases that are not rc to vX-Y-test instead of vX-Y-stable. For their constraints, tag the shared constraints-X-Y branch tip (a beta pins providers at their released versions, which that tip already holds) rather than re-running the identical release-constraints workflow, which is kept for rc and final. --- dev/README_RELEASE_AIRFLOW.md | 19 ++++ .../commands/release_candidate_command.py | 121 ++++++++++++++------- dev/breeze/tests/test_release_candidate_command.py | 120 +++++++++++++++++++- 3 files changed, 217 insertions(+), 43 deletions(-) diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index d29de1bbe39..ee0b8472e67 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -114,6 +114,25 @@ moves `vX-Y-test` forward to the current `main` - only the branch-specific commi `Update default branches for X.Y`) are kept on top of `main`. Each beta is cut from the branch in that state, so everything merged to `main` lands in the next beta. +A beta (`X.Y.0bN`) is cut with the same [`start-rc-process`](#build-rc-artifacts) command as a +release candidate - the only difference is that there is no vote. Because no `vX-Y-stable` branch +exists yet during the beta phase, `start-rc-process` validates, merges and tags against `vX-Y-test` +instead of `vX-Y-stable` when `--version` is a beta (for an `rc` it uses `vX-Y-stable` as before). No +stable branch is required to cut a beta; the stable branch is created at `X.Y.0rc1`. + +Constraints are handled differently for a beta. A beta pins providers at their released versions +from sources that match `main`, so its resolution is already what the shared `constraints-X-Y` +branch holds - `start-rc-process` therefore tags `constraints-X.Y.0bN` at the `constraints-X-Y` +branch tip rather than triggering the `release-constraints` workflow (which only earns its cost for +an `rc`, where the provider wave on PyPI must be pinned, and for a final, which commits onto +`constraints-X-Y`). Sync `constraints-X-Y` to `constraints-main` before cutting the beta so the tip +is current. + +When `start-rc-process` prompts to merge the sync PR for a beta, answer **yes** to merge (so the +beta's version/release-notes commit is created locally and tagged) but **no** to the push - the +commit must not land on `vX-Y-test`, or it would diverge from `main` and break the next +fast-forward. The beta tag therefore references that commit without it being on `vX-Y-test`. + During this phase, all automation that would add commits directly to `vX-Y-test` must be paused - any such commit makes `vX-Y-test` diverge from `main` and breaks the next fast-forward. Upgrades and fixes land on `main` only and reach `vX-Y-test` with the next fast-forward. When you add the new diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index f6ac278bb35..91a33a1b995 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -54,7 +54,7 @@ from airflow_breeze.utils.reproducible import get_source_date_epoch, repack_dete from airflow_breeze.utils.run_utils import run_command from airflow_breeze.utils.shared_options import get_dry_run -RC_PATTERN = re.compile(r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)rc(?P<rc>\d+)$") +PRE_RELEASE_PATTERN = re.compile(r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:b|rc)(?P<pre>\d+)$") SVN_NUM_TRIES = 3 SVN_OPERATION_RETRY_DELAY = 5 @@ -119,13 +119,24 @@ def validate_git_status(): console_print("[success]Working directory is clean") -def validate_version_branches_exist(version_branch, remote_name): +def get_candidate_base_branch(version: str, version_branch: str) -> str: + """Return the branch the candidate is cut, tagged and constrained from. + + An rc uses ``vX-Y-stable``; a beta uses ``vX-Y-test``, which exists before stable does. + """ + from packaging.version import Version + + pre = Version(version).pre + if pre and pre[0] == "rc": + return f"v{version_branch}-stable" + return f"v{version_branch}-test" + + +def validate_version_branches_exist(version_branch, remote_name, candidate_base_branch): """Validate that the required version branches exist.""" console_print(f"[info]Validating version branches exist for {version_branch}...") - # Check if test branch exists test_branch = f"v{version_branch}-test" - stable_branch = f"v{version_branch}-stable" # Fetch to get latest remote branches run_command(["git", "fetch", remote_name], check=True) @@ -139,22 +150,16 @@ def validate_version_branches_exist(version_branch, remote_name): ) remote_branches = result.stdout - test_branch_exists = f"{remote_name}/{test_branch}" in remote_branches - stable_branch_exists = f"{remote_name}/{stable_branch}" in remote_branches - - if not test_branch_exists: - console_print(f"[error]Test branch '{remote_name}/{test_branch}' does not exist!") - console_print("Available remote branches:") - run_command(["git", "branch", "-r"]) - exit(1) - console_print(f"[success]Test branch '{remote_name}/{test_branch}' exists") - - if not stable_branch_exists: - console_print(f"[error]Stable branch '{remote_name}/{stable_branch}' does not exist!") - console_print("Available remote branches:") - run_command(["git", "branch", "-r"]) - exit(1) - console_print(f"[success]Stable branch '{remote_name}/{stable_branch}' exists") + # For a beta candidate_base_branch is the test branch, so this collapses to one check. + candidate_base_role = "Stable branch" if candidate_base_branch.endswith("-stable") else "Test branch" + required_branches = {test_branch: "Test branch", candidate_base_branch: candidate_base_role} + for branch, role in required_branches.items(): + if f"{remote_name}/{branch}" not in remote_branches: + console_print(f"[error]{role} '{remote_name}/{branch}' does not exist!") + console_print("Available remote branches:") + run_command(["git", "branch", "-r"]) + exit(1) + console_print(f"[success]{role} '{remote_name}/{branch}' exists") def validate_tag_does_not_exist(version, remote_name): @@ -199,12 +204,10 @@ def validate_tag_does_not_exist(version, remote_name): exit(1) -def validate_on_correct_branch_for_tagging(version_branch): - """Validate that we're on the correct branch for tagging (stable branch).""" +def validate_on_correct_branch_for_tagging(candidate_base_branch): + """Validate that we're on the correct branch for tagging (the candidate base branch).""" console_print("[info]Validating we're on the correct branch for tagging...") - expected_branch = f"v{version_branch}-stable" - # Check current branch result = run_command( ["git", "branch", "--show-current"], @@ -214,26 +217,26 @@ def validate_on_correct_branch_for_tagging(version_branch): ) current_branch = result.stdout.strip() - if current_branch != expected_branch: - console_print(f"[error]Currently on branch '{current_branch}', expected '{expected_branch}'!") - console_print("Tags should be created on the stable branch after merging the sync PR.") + if current_branch != candidate_base_branch: + console_print(f"[error]Currently on branch '{current_branch}', expected '{candidate_base_branch}'!") + console_print("Tags should be created on the candidate base branch after merging the sync PR.") console_print("Make sure the PR merge step completed successfully.") exit(1) - console_print(f"[success]On correct branch '{expected_branch}' for tagging") + console_print(f"[success]On correct branch '{candidate_base_branch}' for tagging") -def merge_pr(version_branch, remote_name, sync_branch): +def merge_pr(remote_name, sync_branch, candidate_base_branch): if confirm_action("Do you want to merge the Sync PR?"): run_command( [ "git", "checkout", - f"v{version_branch}-stable", + candidate_base_branch, ], check=True, ) run_command( - ["git", "reset", "--hard", f"{remote_name}/v{version_branch}-stable"], + ["git", "reset", "--hard", f"{remote_name}/{candidate_base_branch}"], check=True, ) run_command( @@ -242,7 +245,7 @@ def merge_pr(version_branch, remote_name, sync_branch): ) if confirm_action("Do you want to push the changes? Pushing the changes closes the PR"): run_command( - ["git", "push", remote_name, f"v{version_branch}-stable"], + ["git", "push", remote_name, candidate_base_branch], check=True, ) @@ -443,10 +446,42 @@ def sign_the_release(repo_root): console_print("[success]Release signed") -def generate_and_push_constraints(version, version_branch): - # Resolved from the stable branch the candidate was cut from, so the constraints describe the - # sources being voted on. The workflow reads "rcN" and allows pre-releases accordingly. - publish_constraints(version=version, ref=f"v{version_branch}-stable") +def tag_constraints_from_branch_tip(version, version_branch, remote_name): + """Tag ``constraints-<version>`` at the ``constraints-X-Y`` branch tip. + + A beta pins providers at their released versions, which that tip already holds, so there is + nothing new to resolve. Sync ``constraints-X-Y`` to ``constraints-main`` before cutting so the + tip is current. + """ + constraints_branch = f"constraints-{version_branch}" + constraints_tag = f"constraints-{version}" + if not confirm_action(f"Tag {constraints_tag} at the tip of {remote_name}/{constraints_branch}?"): + return + run_command(["git", "fetch", remote_name, constraints_branch], check=True) + run_command( + [ + "git", + "tag", + "-a", + constraints_tag, + f"{remote_name}/{constraints_branch}", + "-m", + f"Constraints for Apache Airflow {version}", + ], + check=True, + ) + run_command(["git", "push", remote_name, f"refs/tags/{constraints_tag}"], check=True) + console_print(f"[success]Tagged {constraints_tag} at {remote_name}/{constraints_branch} tip") + + +def generate_and_push_constraints(version, version_branch, candidate_base_branch, remote_name): + from packaging.version import Version + + pre = Version(version).pre + if pre and pre[0] == "rc": + publish_constraints(version=version, ref=candidate_base_branch) + else: + tag_constraints_from_branch_tip(version, version_branch, remote_name) def clone_asf_repo(version, repo_root): @@ -647,7 +682,7 @@ def remove_old_releases(version, task_sdk_version, repo_root): if entry.name == version: # Don't remove the current RC continue - if entry.is_dir() and RC_PATTERN.match(entry.name): + if entry.is_dir() and PRE_RELEASE_PATTERN.match(entry.name): old_releases.append(entry.name) old_releases.sort() console_print(f"The following old Airflow releases should be removed: {old_releases}") @@ -669,7 +704,7 @@ def remove_old_releases(version, task_sdk_version, repo_root): if entry.name == task_sdk_version: # Don't remove the current RC continue - if entry.is_dir() and RC_PATTERN.match(entry.name): + if entry.is_dir() and PRE_RELEASE_PATTERN.match(entry.name): old_task_sdk_releases.append(entry.name) old_task_sdk_releases.sort() console_print(f"The following old Task SDK releases should be removed: {old_task_sdk_releases}") @@ -767,6 +802,7 @@ def publish_release_candidate( version_suffix = airflow_version.pre[0] + str(airflow_version.pre[1]) version_branch = str(airflow_version.release[0]) + "-" + str(airflow_version.release[1]) version_without_rc = airflow_version.base_version + candidate_base_branch = get_candidate_base_branch(version, version_branch) task_sdk_version_obj = Version(task_sdk_version) task_sdk_version_without_rc = task_sdk_version_obj.base_version @@ -778,7 +814,7 @@ def publish_release_candidate( console_print("[info]Skipping validations in dry-run mode") validate_remote_tracks_apache_airflow(remote_name) validate_git_status() - validate_version_branches_exist(version_branch, remote_name) + validate_version_branches_exist(version_branch, remote_name, candidate_base_branch) validate_tag_does_not_exist(version, remote_name) validate_tag_does_not_exist(f"task-sdk/{task_sdk_version}", remote_name) @@ -789,6 +825,7 @@ def publish_release_candidate( console_print(f"Task SDK version: {task_sdk_version}") console_print(f"version_suffix: {version_suffix}") console_print(f"version_branch: {version_branch}") + console_print(f"candidate_base_branch: {candidate_base_branch}") console_print(f"version_without_rc: {version_without_rc}") console_print(f"task_sdk_version_without_rc: {task_sdk_version_without_rc}") console_print(f"airflow_repo_root: {airflow_repo_root}") @@ -800,12 +837,12 @@ def publish_release_candidate( console_print() confirm_action("Verify that the above information is correct. Do you want to continue?", abort=True) # Merge the sync PR - merge_pr(version_branch, remote_name, sync_branch) + merge_pr(remote_name, sync_branch, candidate_base_branch) # # # Tag & clean the repo # Validate we're on the correct branch before tagging if not get_dry_run(): - validate_on_correct_branch_for_tagging(version_branch) + validate_on_correct_branch_for_tagging(candidate_base_branch) git_tag(version, f"Apache Airflow {version}") git_tag(f"task-sdk/{task_sdk_version}", f"Airflow Task SDK {task_sdk_version}") git_clean() @@ -829,7 +866,7 @@ def publish_release_candidate( # Sign the release sign_the_release(airflow_repo_root) # Generate, publish and tag the constraints for this candidate - generate_and_push_constraints(version, version_branch) + generate_and_push_constraints(version, version_branch, candidate_base_branch, remote_name) # Clone the asf repo clone_asf_repo(version, airflow_repo_root) # Move artifacts to SVN diff --git a/dev/breeze/tests/test_release_candidate_command.py b/dev/breeze/tests/test_release_candidate_command.py index 8f50ab526bb..4b342997ac3 100644 --- a/dev/breeze/tests/test_release_candidate_command.py +++ b/dev/breeze/tests/test_release_candidate_command.py @@ -45,6 +45,124 @@ def rc_cmd(): return module [email protected]( + ("version", "version_branch", "expected"), + [ + pytest.param("3.4.0rc1", "3-4", "v3-4-stable", id="rc-uses-stable"), + pytest.param("3.4.0rc2", "3-4", "v3-4-stable", id="later-rc-uses-stable"), + pytest.param("3.4.0b1", "3-4", "v3-4-test", id="beta-uses-test"), + ], +) +def test_get_candidate_base_branch(rc_cmd, version, version_branch, expected): + """An rc is cut from the stable branch; a beta from the test branch.""" + assert rc_cmd.get_candidate_base_branch(version, version_branch) == expected + + +def test_generate_and_push_constraints_rc_triggers_workflow(monkeypatch, rc_cmd): + """An rc resolves fresh constraints via the release-constraints workflow, from the stable branch.""" + publish_calls: list[dict] = [] + tag_tip_calls: list[tuple] = [] + + monkeypatch.setattr(rc_cmd, "publish_constraints", lambda **kwargs: publish_calls.append(kwargs)) + monkeypatch.setattr( + rc_cmd, + "tag_constraints_from_branch_tip", + lambda *args: tag_tip_calls.append(args), + ) + + rc_cmd.generate_and_push_constraints("3.4.0rc1", "3-4", "v3-4-stable", "origin") + + assert publish_calls == [{"version": "3.4.0rc1", "ref": "v3-4-stable"}] + assert tag_tip_calls == [] + + +def test_generate_and_push_constraints_beta_tags_branch_tip(monkeypatch, rc_cmd): + """A beta tags the shared constraints branch tip instead of running the workflow.""" + publish_calls: list[dict] = [] + tag_tip_calls: list[tuple] = [] + + monkeypatch.setattr(rc_cmd, "publish_constraints", lambda **kwargs: publish_calls.append(kwargs)) + monkeypatch.setattr( + rc_cmd, + "tag_constraints_from_branch_tip", + lambda *args: tag_tip_calls.append(args), + ) + + rc_cmd.generate_and_push_constraints("3.4.0b1", "3-4", "v3-4-test", "origin") + + assert publish_calls == [] + assert tag_tip_calls == [("3.4.0b1", "3-4", "origin")] + + [email protected]( + ("name", "matches"), + [ + pytest.param("3.4.0rc1", True, id="rc"), + pytest.param("3.4.0b1", True, id="beta"), + pytest.param("3.4.0b2", True, id="later-beta"), + pytest.param("3.4.0", False, id="final"), + pytest.param("3.4.0a1", False, id="alpha"), + pytest.param("not-a-release", False, id="garbage"), + ], +) +def test_pre_release_pattern_matches_rc_and_beta(rc_cmd, name, matches): + """Old-release cleanup collects rc and beta directories, not finals or anything else.""" + assert bool(rc_cmd.PRE_RELEASE_PATTERN.match(name)) is matches + + +def test_remove_old_releases_collects_beta_directories(monkeypatch, rc_cmd): + """A beta run offers to remove older beta directories, not only rc ones.""" + version = "3.4.0b2" + task_sdk_version = "1.5.0b2" + repo_root = "/repo/root" + + entries = [ + FakeDirEntry(version, is_dir=True), # current beta: skipped + FakeDirEntry("3.4.0b1", is_dir=True), # old beta: collected + FakeDirEntry("3.4.0", is_dir=True), # final: excluded + ] + console_messages: list[str] = [] + + def fake_confirm_action(prompt: str, **_kwargs) -> bool: + return prompt == "Do you want to look for old RCs to remove?" + + monkeypatch.setattr(rc_cmd.os, "chdir", lambda path: None) + monkeypatch.setattr(rc_cmd.os, "scandir", lambda: iter(entries)) + monkeypatch.setattr(rc_cmd.os.path, "exists", lambda path: False) + monkeypatch.setattr(rc_cmd, "confirm_action", fake_confirm_action) + monkeypatch.setattr(rc_cmd, "console_print", lambda msg="": console_messages.append(str(msg))) + monkeypatch.setattr(rc_cmd, "run_command", lambda *a, **k: None) + + rc_cmd.remove_old_releases(version=version, task_sdk_version=task_sdk_version, repo_root=repo_root) + + assert "The following old Airflow releases should be removed: ['3.4.0b1']" in console_messages + + +def test_tag_constraints_from_branch_tip_fetches_tags_and_pushes(monkeypatch, rc_cmd): + """Tags constraints-<version> at the remote constraints-X-Y tip and pushes the tag.""" + run_command_calls: list[list[str]] = [] + + monkeypatch.setattr(rc_cmd, "confirm_action", lambda *a, **k: True) + monkeypatch.setattr(rc_cmd, "console_print", lambda *a, **k: None) + monkeypatch.setattr(rc_cmd, "run_command", lambda cmd, **k: run_command_calls.append(cmd)) + + rc_cmd.tag_constraints_from_branch_tip("3.4.0b1", "3-4", "origin") + + assert run_command_calls == [ + ["git", "fetch", "origin", "constraints-3-4"], + [ + "git", + "tag", + "-a", + "constraints-3.4.0b1", + "origin/constraints-3-4", + "-m", + "Constraints for Apache Airflow 3.4.0b1", + ], + ["git", "push", "origin", "refs/tags/constraints-3.4.0b1"], + ] + + def test_clone_asf_repo(monkeypatch, rc_cmd): """Test that clone_asf_repo behaves correctly based on CI environment.""" version = "2.10.0rc3" @@ -148,7 +266,7 @@ def test_remove_old_releases_only_collects_rc_directories(monkeypatch, rc_cmd): # Act rc_cmd.remove_old_releases(version=version, task_sdk_version=task_sdk_version, repo_root=repo_root) - # Assert: only directory entries matching RC_PATTERN, excluding current version, and sorted. + # Assert: only directory entries matching PRE_RELEASE_PATTERN, excluding current version, and sorted. assert f"{repo_root}/asf-dist/dev/airflow" in chdir_calls assert repo_root in chdir_calls assert (
