This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch fork-ci-time-based-status-budget in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 551a4c0cf4ceac8bdeb66fcd4353aeb76624ace9 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 15 23:36:39 2026 +0900 Measure the job status budget between monitor run starts Counting from when reconciling starts credited each run's setup time again, and a re-run credited the time its first attempt already had, so the hourly budget could be exceeded. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq --- .../doc/ci/08_fork_ci_for_external_contributors.md | 10 ++-- .../src/airflow_breeze/utils/fork_ci_reconcile.py | 36 +++++++++---- dev/breeze/tests/test_fork_ci_reconcile.py | 60 +++++++++++++++++----- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md index f9ca257532c..71c4a648a45 100644 --- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md +++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md @@ -1001,11 +1001,13 @@ requests a minute and 500 an hour. Assuming about 35 reported jobs for an averag run (see "The `CI run started` check"), so they do not raise the total; at about 11 pull request runs an hour they take about 1 write a run. -A run gets the hourly budget for the time since the previous monitor run started, not one twelfth -of it. That time is read from the monitor's runs (`GET +A run gets the hourly budget for the time between the previous monitor run's start and its own, +not one twelfth of it. Both are read from the monitor's runs (`GET /repos/apache/airflow/actions/workflows/fork-ci-monitor.yml/runs`, runs still waiting in its -concurrency group left out), not from the record the monitor carries between runs, which a failed -run or a failed Slack post leaves unchanged although that run may have posted statuses. GitHub +concurrency group left out), so they are the same boundary of each run, setup included, and not +from the record the monitor carries between runs, which a failed run or a failed Slack post leaves +unchanged although that run may have posted statuses. A re-run counts from when its run was +created, so the time its first attempt had is not credited again. GitHub starts scheduled workflows later than their cron: on 2026-11-15 the 5-minute monitor ran every 9 to 10 minutes, so a fixed 13 writes a run (160 an hour at the cron's cadence) posted only about 82 an hour, and four full-matrix fork runs at once took about six hours to catch up. Runs queued diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py index d1501093389..010edf8e05d 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py @@ -2831,16 +2831,21 @@ def read_run_count(reported_file: Path | None, key: str) -> int | None: return count if isinstance(count, int) and not isinstance(count, bool) else 0 -def find_previous_monitor_run_start(client: GitHubClient, repo: str, *, run_url: str) -> datetime | None: +def find_time_since_previous_monitor_run( + client: GitHubClient, repo: str, *, run_url: str +) -> timedelta | None: """ - When the monitor's latest run other than this one started; None when it cannot be told. + The time between the previous monitor run's start and this run's; None when it cannot be told. - Read from GitHub rather than from the record the monitor carries between runs, which a failed run - or a failed Slack post leaves unchanged although that run may have posted statuses. + Both starts are read from GitHub, so they are the same boundary of each run, setup included, and a + failed run or a failed Slack post, which leave the record the monitor carries between runs + unchanged, still counts. A re-run of this run counts from when the run was created, the earliest + its first attempt can have posted. """ match = re.search(r"/actions/runs/(\d+)", run_url) if match is None: return None + run_id = int(match[1]) try: runs = client.rest_get( f"repos/{repo}/actions/workflows/{MONITOR_WORKFLOW_FILE}/runs?per_page={MONITOR_RUNS_READ}" @@ -2848,13 +2853,25 @@ def find_previous_monitor_run_start(client: GitHubClient, repo: str, *, run_url: except Exception as e: console_print(f"[warning]Could not read the monitor's earlier runs: {e}[/]") return None + current = next((run for run in runs if run["id"] == run_id), None) + started = parse_github_datetime(current.get("run_started_at")) if current else None + if current is None or started is None: + return None # Runs waiting in the concurrency group have not started, so they posted nothing. - starts = [ - parse_github_datetime(run.get("run_started_at")) + boundaries = [ + start for run in runs - if run["id"] != int(match[1]) and run.get("status") not in NOT_STARTED_RUN_STATUSES + if run["id"] != run_id and run.get("status") not in NOT_STARTED_RUN_STATUSES + for start in [parse_github_datetime(run.get("run_started_at"))] + if start is not None and start <= started ] - return max((start for start in starts if start is not None), default=None) + if (current.get("run_attempt") or 1) > 1: + created = parse_github_datetime(current.get("created_at")) + if created is not None: + boundaries.append(created) + if not boundaries: + return None + return started - max(boundaries) def read_job_status_failure_runs(reported_file: Path | None) -> dict[int, int]: @@ -2985,9 +3002,8 @@ def run_reconcile( f"listing the pull requests with `{LABEL_USE_PROJECT_CI}` failed: {type(e).__name__}" ) read_jobs: ReadJobs = {} - previous_run_at = find_previous_monitor_run_start(client, repo, run_url=run_url) job_status_budget = compute_job_status_budget( - now - previous_run_at if previous_run_at is not None else None + find_time_since_previous_monitor_run(client, repo, run_url=run_url) ) job_reporter = JobStatusReporter( client, diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py b/dev/breeze/tests/test_fork_ci_reconcile.py index 51221295180..7c850503798 100644 --- a/dev/breeze/tests/test_fork_ci_reconcile.py +++ b/dev/breeze/tests/test_fork_ci_reconcile.py @@ -87,11 +87,11 @@ from airflow_breeze.utils.fork_ci_reconcile import ( find_failed_jobs, find_heads_shared_across_bases, find_passed_jobs, - find_previous_monitor_run_start, find_project_ci_label_heads, find_project_ci_passed_url, find_project_ci_run_url, find_state_comment, + find_time_since_previous_monitor_run, is_cancelled_gate_run, is_gated_skip_run, is_own_pilot_change, @@ -3462,8 +3462,14 @@ def test_job_status_budget_after_a_gap_is_capped(): MONITOR_RUN_URL = "https://github.com/apache/airflow/actions/runs/100" -def _monitor_run(run_id, started, status="completed"): - return {"id": run_id, "status": status, "run_started_at": started} +def _monitor_run(run_id, started, status="completed", attempt=1, created=None): + return { + "id": run_id, + "status": status, + "run_started_at": started, + "run_attempt": attempt, + "created_at": created or started, + } @pytest.mark.parametrize( @@ -3476,33 +3482,63 @@ def _monitor_run(run_id, started, status="completed"): _monitor_run(99, "2026-11-10T11:51:00Z"), _monitor_run(98, "2026-11-10T11:42:00Z"), ], - datetime(2026, 11, 10, 11, 51, tzinfo=UTC), + timedelta(minutes=7), id="previous-started-run", ), + pytest.param( + [ + _monitor_run(101, "2026-11-10T11:59:00Z"), + _monitor_run( + 100, + "2026-11-10T12:05:00Z", + status="in_progress", + attempt=2, + created="2026-11-10T11:58:00Z", + ), + _monitor_run(99, "2026-11-10T11:51:00Z"), + ], + timedelta(minutes=6), + id="re-run-after-a-later-run", + ), + pytest.param( + [ + _monitor_run( + 100, + "2026-11-10T12:05:00Z", + status="in_progress", + attempt=2, + created="2026-11-10T11:58:00Z", + ), + _monitor_run(99, "2026-11-10T11:51:00Z"), + ], + timedelta(minutes=7), + id="re-run-counts-from-its-first-attempt", + ), pytest.param([_monitor_run(100, "2026-11-10T11:58:00Z")], None, id="first-run"), + pytest.param([_monitor_run(99, "2026-11-10T11:51:00Z")], None, id="this-run-not-listed"), ], ) -def test_find_previous_monitor_run_start(client, runs, expected): +def test_find_time_since_previous_monitor_run(client, runs, expected): client.rest_get.return_value = {"workflow_runs": runs} - assert find_previous_monitor_run_start(client, REPO, run_url=MONITOR_RUN_URL) == expected + assert find_time_since_previous_monitor_run(client, REPO, run_url=MONITOR_RUN_URL) == expected client.rest_get.assert_called_once_with( f"repos/{REPO}/actions/workflows/fork-ci-monitor.yml/runs?per_page=10" ) -def test_find_previous_monitor_run_start_without_a_run_id(client): - assert find_previous_monitor_run_start(client, REPO, run_url="https://run") is None +def test_find_time_since_previous_monitor_run_without_a_run_id(client): + assert find_time_since_previous_monitor_run(client, REPO, run_url="https://run") is None client.rest_get.assert_not_called() -def test_find_previous_monitor_run_start_when_the_runs_cannot_be_read(client): +def test_find_time_since_previous_monitor_run_when_the_runs_cannot_be_read(client): client.rest_get.side_effect = GitHubError("gh api failed: HTTP 502") - assert find_previous_monitor_run_start(client, REPO, run_url=MONITOR_RUN_URL) is None + assert find_time_since_previous_monitor_run(client, REPO, run_url=MONITOR_RUN_URL) is None [email protected]("airflow_breeze.utils.fork_ci_reconcile.find_previous_monitor_run_start", autospec=True) [email protected]("airflow_breeze.utils.fork_ci_reconcile.find_time_since_previous_monitor_run", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.JobStatusReporter", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.report_project_ci_runs_started", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_reconcile.classify", autospec=True) @@ -3514,7 +3550,7 @@ def test_run_reconcile_budgets_job_statuses_for_the_time_since_the_previous_run( mock_reporter.return_value.report.return_value = None mock_reporter.return_value.deferred = [] mock_reporter.return_value.delayed = [] - mock_previous.return_value = NOW - timedelta(minutes=9) + mock_previous.return_value = timedelta(minutes=9) _reconcile_with_reporter(client, tmp_path)
