This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fork-ci-time-based-status-budget in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e50b1c02fa32143ea75c74802af1285b1ac47dca Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 15 23:49:10 2026 +0900 Keep the passed-jobs count when the fork status write fails The state comment records a completed run before the status is posted, and later runs do not read that run's jobs again, so a failed status write would have lost the count for good. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq --- .../doc/ci/08_fork_ci_for_external_contributors.md | 11 ++++--- .../src/airflow_breeze/utils/fork_ci_messages.py | 3 +- .../src/airflow_breeze/utils/fork_ci_reconcile.py | 27 ++++++++++----- dev/breeze/tests/test_fork_ci_reconcile.py | 38 +++++++++++++--------- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index 09c935aaf21..1350c7641b9 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -883,8 +883,10 @@ observed. The `description` is a short sentence under 140 characters that always where to look next. For `running`, `failed`, `green` and `green_unverified` it is followed by the fork run's passed jobs, such as "Fork CI passed - see Details (127 jobs passed)", since passed jobs are not posted as statuses of their own (see "Job statuses"). The count is taken from the jobs the -run read for its job statuses; a run that did not read them keeps the count the status shows, and -the count is never carried over from another state. The reconciler reads the state back from the +run read for its job statuses, or else from the marker of a completed run whose jobs are all +reported, so a status write that failed after the state comment recorded the run still gets it; +otherwise the status keeps the count it shows, and the count is never carried over from another +state. The reconciler reads the state back from the description without the count, and accepts a count only on those four states. Statuses are immutable and per SHA. The reconciler compares the desired `(state, @@ -940,8 +942,9 @@ suites); its jobs come from `GET /repos/<fork>/actions/runs/<id>/jobs?filter=lat Fork CI / Tests (AMD)", so it no longer shows as running or failed; one that is already a success is left as it is. - Once every job of a completed run is reported, the run is recorded in the marker as - `jobs: "<run id>:<attempt>:<conclusion>"` (`runAttempt` is fetched with the check suites), so - later runs do not read its jobs again. A run in progress is read on every run, and a re-run in + `jobs: "<run id>:<attempt>:<conclusion>:<passed jobs>"` (`runAttempt` is fetched with the check + suites), so later runs do not read its jobs again. A marker recorded before the count was added + has none, and its run's jobs are read once more. A run in progress is read on every run, and a re-run in the fork is read again because its attempt changes, even when it ends with the same conclusion. - Reporting job statuses is caught on its own: when it fails, the pull request is still classified, so for example a `CI passed` that no longer holds is still revoked. Reading a fork diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py index b313399a798..c6b63b921c0 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_messages.py @@ -120,7 +120,8 @@ class Marker: since: datetime # Set when the head commit's fork CI turned green and its base could be compared. base: BaseCheck | None = None - # The completed fork run (``<run id>:<attempt>:<conclusion>``) whose jobs are all reported. + # The completed fork run (``<run id>:<attempt>:<conclusion>:<passed jobs>``) whose jobs are all + # reported. jobs: str | None = None # The fork run (``<run id>:<attempt>``) whose failed jobs the comment lists. failed_run: str | None = None diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index 49cc0167f99..5884dfabf77 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -1891,10 +1891,18 @@ def count_passed_jobs(jobs: Iterable[dict[str, Any]]) -> int: ) -def find_passed_jobs(fork: ForkObservation, read_jobs: ReadJobs) -> int | None: - """The passed jobs of the fork run, when this reconciler run read its jobs.""" - jobs = read_jobs.get(fork.run_id) if fork.run_id is not None else None - return count_passed_jobs(jobs) if jobs is not None else None +def find_passed_jobs(fork: ForkObservation, read_jobs: ReadJobs, jobs_reported: str | None) -> int | None: + """ + The passed jobs of the fork run, from its jobs when this reconciler run read them, or else from + the key of the completed run whose jobs are all reported. + """ + if fork.run_id is None: + return None + jobs = read_jobs.get(fork.run_id) + if jobs is not None: + return count_passed_jobs(jobs) + *_, passed = (jobs_reported or "").split(":") + return int(passed) if passed.isdigit() else None def build_job_statuses(jobs: Iterable[dict[str, Any]]) -> dict[str, tuple[str, str, str]]: @@ -2016,7 +2024,7 @@ class JobStatusReporter: The key is recorded in the marker, so a completed run's jobs are read once. A run in progress, or one whose statuses did not all fit in the budget, is read again next time. The key holds the run attempt, so a re-run in the fork is read again even when it ends with the - same conclusion. + same conclusion, and the run's passed jobs, which later runs show without reading the jobs. """ if fork.run_id is None or not pr.head_repo: return None @@ -2026,8 +2034,9 @@ class JobStatusReporter: else None ) recorded = pr.marker.jobs if pr.marker is not None and pr.marker.sha == pr.head_sha else None - if key is not None and key == recorded: - return key + # Recorded before the passed jobs were, a key has no count; its jobs are then read once more. + if key is not None and recorded is not None and recorded.startswith(f"{key}:"): + return recorded # Without writes left, the reads are spent for nothing: a backlog of hundreds of pull requests # would otherwise cost two reads each on every run. if self.remaining == 0: @@ -2082,7 +2091,7 @@ class JobStatusReporter: self._client.set_status(pr.number, pr.head_sha, state, target_url, description, context=context) # Charged per write, so writes made before a later one fails still count. self.remaining -= 1 - return key + return None if key is None else f"{key}:{count_passed_jobs(jobs)}" def fetch_workflow_state(client: GitHubClient, fork: str) -> str: @@ -3070,7 +3079,7 @@ def run_reconcile( verify_head_repo_unavailable=functools.partial( check_head_repo_unavailable, client, repo, pr.number ), - passed_jobs=find_passed_jobs(fork, read_jobs), + passed_jobs=find_passed_jobs(fork, read_jobs, jobs_reported), ) # A revocation to `pending` always goes through: it is idempotent and must not depend on # `report_project_ci_passed` succeeding later in this run. diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index fb50f20fb23..59f1f78d806 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -3136,7 +3136,7 @@ def test_job_status_reporter_posts_changed_statuses_failures_first(client): key = JobStatusReporter(client, repo=REPO).report(_pr(), COMPLETED_RUN) - assert key == "1:1:FAILURE" + assert key == "1:1:FAILURE:2" assert [(c.args[2], c.kwargs["context"]) for c in client.set_status.call_args_list] == [ ("failure", f"{JOB_STATUS_CONTEXT_PREFIX}Unit"), ("success", f"{JOB_STATUS_CONTEXT_PREFIX}Helm tests"), @@ -3167,19 +3167,22 @@ def test_job_status_reporter_does_not_record_a_run_in_progress(client): @pytest.mark.parametrize( - ("marker_jobs", "marker_sha", "reads"), + ("marker_jobs", "marker_sha", "reads", "expected"), [ - pytest.param("1:1:FAILURE", "abc123", False, id="already-reported"), - pytest.param("1:1:SUCCESS", "abc123", True, id="re-run-with-another-conclusion"), - pytest.param("1:2:FAILURE", "abc123", True, id="another-attempt-with-the-same-conclusion"), - pytest.param("1:1:FAILURE", "other", True, id="marker-of-another-commit"), + pytest.param("1:1:FAILURE:7", "abc123", False, "1:1:FAILURE:7", id="already-reported"), + pytest.param("1:1:FAILURE", "abc123", True, "1:1:FAILURE:0", id="recorded-before-the-count"), + pytest.param("1:1:SUCCESS:7", "abc123", True, "1:1:FAILURE:0", id="re-run-with-another-conclusion"), + pytest.param( + "1:2:FAILURE:7", "abc123", True, "1:1:FAILURE:0", id="another-attempt-with-the-same-conclusion" + ), + pytest.param("1:1:FAILURE:7", "other", True, "1:1:FAILURE:0", id="marker-of-another-commit"), ], ) -def test_job_status_reporter_reads_a_completed_run_once(client, marker_jobs, marker_sha, reads): +def test_job_status_reporter_reads_a_completed_run_once(client, marker_jobs, marker_sha, reads, expected): _reporter_client(client, []) pr = _pr(marker=dataclasses.replace(_marker("failed", sha=marker_sha), jobs=marker_jobs)) - assert JobStatusReporter(client, repo=REPO).report(pr, COMPLETED_RUN) == "1:1:FAILURE" + assert JobStatusReporter(client, repo=REPO).report(pr, COMPLETED_RUN) == expected assert client.rest_get.called is reads @@ -3568,16 +3571,19 @@ def test_count_passed_jobs(): @pytest.mark.parametrize( - ("fork", "read_jobs", "expected"), + ("fork", "read_jobs", "jobs_reported", "expected"), [ - pytest.param(COMPLETED_RUN, {1: [_fork_job("A")]}, 1, id="read"), - pytest.param(COMPLETED_RUN, {}, None, id="not-read"), - pytest.param(COMPLETED_RUN, {1: None}, None, id="read-failed"), - pytest.param(ForkObservation(), {1: [_fork_job("A")]}, None, id="no-run"), + pytest.param(COMPLETED_RUN, {1: [_fork_job("A")]}, None, 1, id="read"), + pytest.param(COMPLETED_RUN, {1: [_fork_job("A")]}, "1:1:FAILURE:9", 1, id="read-wins-over-the-key"), + pytest.param(COMPLETED_RUN, {}, "1:1:FAILURE:9", 9, id="from-the-recorded-key"), + pytest.param(COMPLETED_RUN, {}, "1:1:FAILURE", None, id="key-recorded-before-the-count"), + pytest.param(COMPLETED_RUN, {}, None, None, id="not-read"), + pytest.param(COMPLETED_RUN, {1: None}, None, None, id="read-failed"), + pytest.param(ForkObservation(), {1: [_fork_job("A")]}, "1:1:FAILURE:9", None, id="no-run"), ], ) -def test_find_passed_jobs(fork, read_jobs, expected): - assert find_passed_jobs(fork, read_jobs) == expected +def test_find_passed_jobs(fork, read_jobs, jobs_reported, expected): + assert find_passed_jobs(fork, read_jobs, jobs_reported) == expected def test_build_job_statuses_treats_contexts_case_insensitively(): @@ -3599,7 +3605,7 @@ def test_job_status_reporter_compares_contexts_case_insensitively(client): } _reporter_client(client, [_fork_job("Tests", conclusion="failure")], [posted]) - assert JobStatusReporter(client, repo=REPO).report(_pr(), COMPLETED_RUN) == "1:1:FAILURE" + assert JobStatusReporter(client, repo=REPO).report(_pr(), COMPLETED_RUN) == "1:1:FAILURE:0" client.set_status.assert_not_called()
