potiuk commented on code in PR #74823: URL: https://github.com/apache/airflow/pull/74823#discussion_r4367844259
########## dev/breeze/src/airflow_breeze/utils/git_remotes.py: ########## @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Git remotes of the local checkout, as `breeze ci upgrade` and `breeze ci audit` understand them.""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass + +from airflow_breeze.utils.run_utils import run_command + +# Every form git and GitHub accept for a GitHub repository: http or https with optional +# credentials, `www.` or a port, scp-like ssh and ssh URLs with an optional port. The host is +# case-insensitive. An ssh host may also be an alias from ~/.ssh/config, such as +# `git@github-personal:apache/airflow.git`: a host without a dot that names `github`. A dotted +# host other than github.com, such as a GitHub Enterprise server, is a different GitHub. The +# repository must be the whole path, so `apache/airflow-site` never matches `apache/airflow`. +_SSH_GITHUB_HOST = r"(?:(?:www\.)?github\.com|[^@\s/:.]*github[^@\s/:.]*)" Review Comment: Fixed: `github.com-<suffix>` is now accepted as an ssh host alias. Both `[email protected]:` and `ssh://[email protected]/` are in `test_get_github_repo`, and the doc mentions the form. The suffix excludes dots, so `github.com-work.example` stays a different host. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/src/airflow_breeze/utils/ci_audit.py: ########## @@ -0,0 +1,382 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Checks behind `breeze ci audit`: is this checkout and fork set up for fork CI (AIP-120)? + +Every check is read-only. A failing check offers the commands that fix it; `--fix` runs them after +confirmation. See "breeze ci audit" in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +from packaging.version import Version + +from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, MIN_GH_VERSION +from airflow_breeze.utils.git_remotes import find_remote, find_remote_for_repo, parse_remotes + +if TYPE_CHECKING: + from airflow_breeze.utils.fork_ci_gate import GateConfig + +UPSTREAM_URL = f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git"; +CI_WORKFLOW_FILE = "ci-amd.yml" +# Commits a fork's main may lag behind upstream main before a sync is suggested. +MAX_FORK_MAIN_BEHIND = 200 + +Runner = Callable[[Sequence[str]], subprocess.CompletedProcess] + + +class CheckStatus(str, Enum): + OK = "ok" + FAIL = "fail" + INFO = "info" + + +class Role(str, Enum): + COMMITTER = "committer" + COLLABORATOR = "collaborator" + EXEMPTED = "exempted organisation member" + PILOT = "fork CI pilot participant" + EXTERNAL = "external contributor" + + +@dataclass(frozen=True) +class CheckResult: + number: int + name: str + status: CheckStatus + finding: str + fix: tuple[str, ...] = () + + +def run_tool(command: Sequence[str]) -> subprocess.CompletedProcess: + """Run a command for the audit; a missing executable is a failed command, not a crash.""" + try: + return subprocess.run(list(command), capture_output=True, text=True, check=False) + except OSError as e: + return subprocess.CompletedProcess(list(command), 127, stdout="", stderr=str(e)) + + +def _ok(result: subprocess.CompletedProcess) -> bool: + return result.returncode == 0 + + +def _out(result: subprocess.CompletedProcess) -> str: + return (result.stdout or "").strip() + + +def redact_url(url: str) -> str: + """Drop the credentials an http(s) remote URL may carry, so a finding never prints a token.""" + return re.sub(r"^(https?://)[^@/]+@", r"\1", url) + + +def _is_not_found(result: subprocess.CompletedProcess) -> bool: + return "404" in (result.stderr or "") or "Not Found" in (result.stderr or "") + + +def is_public_apache_member(*, login: str, run: Runner) -> bool: + return _ok(run(["gh", "api", f"orgs/apache/public_members/{login}"])) Review Comment: Fixed: only a 404 means "not a public member". Any other failure gives a role that couldn't be checked (`Role.UNKNOWN`), which keeps checks 2, 3 and 5 informational, so `--fix` no longer offers `gh workflow enable`. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/src/airflow_breeze/utils/ci_audit.py: ########## @@ -0,0 +1,382 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Checks behind `breeze ci audit`: is this checkout and fork set up for fork CI (AIP-120)? + +Every check is read-only. A failing check offers the commands that fix it; `--fix` runs them after +confirmation. See "breeze ci audit" in dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +from packaging.version import Version + +from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, MIN_GH_VERSION +from airflow_breeze.utils.git_remotes import find_remote, find_remote_for_repo, parse_remotes + +if TYPE_CHECKING: + from airflow_breeze.utils.fork_ci_gate import GateConfig + +UPSTREAM_URL = f"https://github.com/{APACHE_AIRFLOW_GITHUB_REPOSITORY}.git"; +CI_WORKFLOW_FILE = "ci-amd.yml" +# Commits a fork's main may lag behind upstream main before a sync is suggested. +MAX_FORK_MAIN_BEHIND = 200 + +Runner = Callable[[Sequence[str]], subprocess.CompletedProcess] + + +class CheckStatus(str, Enum): + OK = "ok" + FAIL = "fail" + INFO = "info" + + +class Role(str, Enum): + COMMITTER = "committer" + COLLABORATOR = "collaborator" + EXEMPTED = "exempted organisation member" + PILOT = "fork CI pilot participant" + EXTERNAL = "external contributor" + + +@dataclass(frozen=True) +class CheckResult: + number: int + name: str + status: CheckStatus + finding: str + fix: tuple[str, ...] = () + + +def run_tool(command: Sequence[str]) -> subprocess.CompletedProcess: + """Run a command for the audit; a missing executable is a failed command, not a crash.""" + try: + return subprocess.run(list(command), capture_output=True, text=True, check=False) + except OSError as e: + return subprocess.CompletedProcess(list(command), 127, stdout="", stderr=str(e)) + + +def _ok(result: subprocess.CompletedProcess) -> bool: + return result.returncode == 0 + + +def _out(result: subprocess.CompletedProcess) -> str: + return (result.stdout or "").strip() + + +def redact_url(url: str) -> str: + """Drop the credentials an http(s) remote URL may carry, so a finding never prints a token.""" + return re.sub(r"^(https?://)[^@/]+@", r"\1", url) + + +def _is_not_found(result: subprocess.CompletedProcess) -> bool: + return "404" in (result.stderr or "") or "Not Found" in (result.stderr or "") + + +def is_public_apache_member(*, login: str, run: Runner) -> bool: + return _ok(run(["gh", "api", f"orgs/apache/public_members/{login}"])) + + +def is_private_apache_member(*, run: Runner) -> bool: + membership = run(["gh", "api", "user/memberships/orgs/apache", "--jq", ".state"]) + return _ok(membership) and _out(membership) == "active" + + +def get_role(*, login: str, run: Runner, config: GateConfig, collaborators: frozenset[str]) -> Role: + # In the gate's order: a pilot entry gates a pull request whatever its author's role. + if config.pilot_joined_at(login) is not None: + return Role.PILOT + # The gate only sees public membership: GitHub reports a private member's pull requests as + # `CONTRIBUTOR`, so they are gated like an external contributor's. + if is_public_apache_member(login=login, run=run): + return Role.COMMITTER + if login.casefold() in {c.casefold() for c in collaborators}: + return Role.COLLABORATOR + if config.is_allowlisted(login): + return Role.EXEMPTED + return Role.EXTERNAL + + +def fork_ci_required(role: Role, config: GateConfig) -> bool | None: + """True when fork CI must be on, False when it must be off, None when either is fine.""" + if role == Role.PILOT: + return True + if role == Role.EXTERNAL: + return True if config.enabled else None + return False + + +def run_audit(*, run: Runner, config: GateConfig, collaborators: frozenset[str]) -> list[CheckResult]: + results: list[CheckResult] = [] + + gh_version = run(["gh", "--version"]) + if not _ok(gh_version): + results.append( + CheckResult( + 1, "gh", CheckStatus.FAIL, "`gh` is not installed", ("echo See https://cli.github.com";,) + ) + ) + return results + version_match = re.search(r"gh version (\d+\.\d+\.\d+)", _out(gh_version)) + if version_match is None or Version(version_match.group(1)) < Version(MIN_GH_VERSION): + results.append( + CheckResult(1, "gh", CheckStatus.FAIL, f"`gh` is older than {MIN_GH_VERSION}; upgrade it") + ) + return results + user = run(["gh", "api", "user", "--jq", ".login"]) + if not _ok(user): + results.append( + CheckResult(1, "gh", CheckStatus.FAIL, "`gh` is not authenticated", ("gh auth login",)) + ) + return results + login = _out(user) + results.append(CheckResult(1, "gh", CheckStatus.OK, f"authenticated as {login}")) + + # The role decides whether the fork setup matters at all: a committer working from an + # apache/airflow clone without a fork is fine while their pull requests run project CI. + role = get_role(login=login, run=run, config=config, collaborators=collaborators) + required = fork_ci_required(role, config) + setup_status = CheckStatus.INFO if required is False else CheckStatus.FAIL + setup_note = "; only needed for fork CI, which your pull requests do not use" if required is False else "" + + remotes = parse_remotes(_out(run(["git", "remote", "-v"]))) + upstream = find_remote(remotes, "upstream") + apache = find_remote_for_repo(remotes, APACHE_AIRFLOW_GITHUB_REPOSITORY) + origin = find_remote(remotes, "origin") + # Whether the remotes fix already adds `origin`, so the fork fix must not add it again. + adds_origin = False + if ( + upstream is not None + and upstream.repo + and upstream.repo.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY + ): + if origin is not None and origin.repo and origin.repo.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY: + results.append( + CheckResult( + 2, + "remotes", + setup_status, + f"`origin` points at apache/airflow; it should be your fork{setup_note}", + (f"git remote set-url origin https://github.com/{login}/airflow.git";,), + ) + ) + else: + results.append(CheckResult(2, "remotes", CheckStatus.OK, "`upstream` is apache/airflow")) + elif apache is not None and upstream is None: + fix = [f"git remote rename {apache.name} upstream"] + if apache.name == "origin": + fix.append(f"git remote add origin https://github.com/{login}/airflow.git";) + adds_origin = True + results.append( + CheckResult( + 2, + "remotes", + setup_status, + f"apache/airflow is the remote `{apache.name}`; Airflow's convention is `upstream`{setup_note}", + tuple(fix), + ) + ) + elif upstream is not None: + results.append( + CheckResult( + 2, + "remotes", + CheckStatus.FAIL, + f"`upstream` points at {redact_url(upstream.url)}, not apache/airflow", + (f"git remote set-url upstream {UPSTREAM_URL}",), + ) + ) + else: + results.append( + CheckResult( + 2, + "remotes", + CheckStatus.FAIL, + "no `upstream` remote pointing at apache/airflow", + (f"git remote add upstream {UPSTREAM_URL}",), + ) + ) + + fork = origin.repo if origin is not None and origin.repo else None + fork_ok = False + if fork is None or fork.casefold() == APACHE_AIRFLOW_GITHUB_REPOSITORY: + results.append( + CheckResult( + 3, + "fork", + setup_status, + f"no `origin` remote pointing at your fork of apache/airflow{setup_note}", + ( + f"gh repo fork {APACHE_AIRFLOW_GITHUB_REPOSITORY} --remote=false", + *( + () + if adds_origin + else (f"git remote add origin https://github.com/{login}/airflow.git";,) Review Comment: Fixed: the `origin points at apache/airflow` branch now sets `adds_origin`, and a test checks that check 3 no longer offers `git remote add origin`. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting ########## dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md: ########## @@ -763,19 +763,36 @@ prints one row: status, finding, and the command that fixes it. |---|---|---|---| | 1 | `gh` present and authenticated | `gh auth status` succeeds, version at least `MIN_GH_VERSION` | install / `gh auth login` | Review Comment: Fixed: the rows now say `gh api user` and "more than 200 commits (`MAX_FORK_MAIN_BEHIND`)", and "currently" is gone. --- Drafted-by: Claude Code (Opus 5); reviewed by @potiuk before posting -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
