This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new df9fac6008f Say why a fork CI monitor call failed in its Slack note
(#77132)
df9fac6008f is described below
commit df9fac6008fa7e308597cada08543f4fcb447aa4
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Nov 17 20:46:11 2026 +0100
Say why a fork CI monitor call failed in its Slack note (#77132)
* Say why a fork CI monitor call failed in its Slack note
Notes such as "reporting `CI run started` for project CI failed" named
only the exception type, so a failure seen three times in the pilot
could not be diagnosed from Slack.
Generated-by: Claude Opus 5
Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq
* Keep the cause of retried and timed-out GitHub calls in Slack notes
Only "failed:" messages had the gh arguments removed, so for a retried,
rate-limited or timed-out call the arguments could use up the note's
200 characters and leave out what went wrong.
Generated-by: Claude Opus 5
Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq
---
.../doc/ci/08_fork_ci_for_external_contributors.md | 6 ++-
.../src/airflow_breeze/utils/fork_ci_reconcile.py | 34 ++++++++++++---
dev/breeze/tests/test_fork_ci_reconcile.py | 50 ++++++++++++++++++++--
3 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
index f5346e8078a..735c340f25d 100644
--- a/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
+++ b/dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md
@@ -1328,7 +1328,11 @@ Unknown means any of:
- A state comment whose marker cannot be parsed.
Expected states never alert. The message lists the pull request numbers and
the reason for
-each, with a link to the workflow run, so a maintainer can act. The run itself
still ends
+each, with a link to the workflow run, so a maintainer can act. An item or
note about a failed
+call names the exception and the first line of its message, shortened to 200
characters; a failed
+`gh` call's arguments, which for GraphQL hold the whole query, are left out,
and the run log has
+the error in full. A failure that persists gives the same text on every run,
so it is still posted
+once. The run itself still ends
green unless the reconciler could not talk to GitHub at all.
### Budget
diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
index ba8432768b6..cb56eacdf37 100644
--- a/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
+++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
@@ -202,6 +202,13 @@ COMPARE_FILES_CAP = 300
SLACK_SECTION_LIMIT = 3000
SLACK_MAX_BLOCKS = 50
SLACK_ITEM_LIMIT = 500
+# The error text a Slack item quotes; the full error is in the run log.
+ERROR_TEXT_LIMIT = 200
+# How GitHubClient ends a failed `gh` call's message, after the call's
arguments.
+GH_FAILURE = re.compile(
+ r" (?:failed: |(?=still failing after \d+ attempts: )|(?=still rate
limited after \d+ attempts)"
+ r"|(?=timed out after \d+s))"
+)
# Events whose runs use the workflow files of the pull request's merge commit,
so a pull request
# can change what they run. pull_request_target and issue_comment use the base
branch's files.
MERGE_REF_EVENTS = ("pull_request", "pull_request_review",
"pull_request_review_comment")
@@ -591,6 +598,23 @@ def changed_paths(entries: Iterable[dict[str, Any]]) ->
list[str]:
return paths
+def describe_error(e: Exception) -> str:
+ """
+ The exception type and the first line of its message, for a Slack item.
+
+ A failed `gh` call's message starts with its arguments, which for GraphQL
hold the whole query,
+ so only what `gh` reported is kept. A failure that persists gets the same
text on every run, so
+ its item is still posted once.
+ """
+ message = str(e).strip()
+ failure = GH_FAILURE.search(message) if message.startswith("gh ") else None
+ if failure:
+ message = message[failure.end() :]
+ first_line = message.splitlines()[0].strip() if message else ""
+ text = f"{type(e).__name__}: {first_line}" if first_line else
type(e).__name__
+ return text if len(text) <= ERROR_TEXT_LIMIT else f"{text[:
ERROR_TEXT_LIMIT - 1]}…"
+
+
def changes_ci_definition(files: Sequence[str], *, file_count: int | None =
None) -> bool:
"""
Whether a pull request's changed files can alter what its fork CI runs.
@@ -1560,7 +1584,7 @@ def report_project_ci_passed(
successes += change[0] == "success"
except Exception as e:
console_print(f"[error]#{node['number']}: `{CI_PASSED_CONTEXT}`
not updated: {e}[/]")
- attention.append((node["number"], f"`{CI_PASSED_CONTEXT}` not
updated: {type(e).__name__}"))
+ attention.append((node["number"], f"`{CI_PASSED_CONTEXT}` not
updated: {describe_error(e)}"))
return written
@@ -1744,7 +1768,7 @@ def discover(
except Exception as e:
console_print(f"[error]#{node['number']}: `{CI_PASSED_CONTEXT}`
not updated: {e}[/]")
if unknown is not None:
- unknown.append((node["number"], f"`{CI_PASSED_CONTEXT}` not
updated: {type(e).__name__}"))
+ unknown.append((node["number"], f"`{CI_PASSED_CONTEXT}` not
updated: {describe_error(e)}"))
for search_query in queries:
for node in _search(client, DISCOVERY_QUERY % {"context":
CI_PASSED_CONTEXT}, search_query):
@@ -3138,7 +3162,7 @@ def run_reconcile(
# Writes made before the failure are not known, so the whole allowance
is charged.
ci_run_started_writes = MAX_CI_RUN_STARTED_PER_RUN
ci_run_started_note = (
- f"reporting `{CI_RUN_STARTED_CONTEXT}` for project CI failed:
{type(e).__name__}"
+ f"reporting `{CI_RUN_STARTED_CONTEXT}` for project CI failed:
{describe_error(e)}"
)
# Commit statuses belong to a commit, so a fork `success` posted for any
pull request on one of
# these would also let a pull request with `use project ci` merge. None
when they could not be
@@ -3153,7 +3177,7 @@ def run_reconcile(
project_ci_heads = None
ci_passed_reserved = MAX_PROJECT_CI_PASSED_PER_RUN
ci_passed_notes.append(
- f"listing the pull requests with `{LABEL_USE_PROJECT_CI}` failed:
{type(e).__name__}"
+ f"listing the pull requests with `{LABEL_USE_PROJECT_CI}` failed:
{describe_error(e)}"
)
read_jobs: ReadJobs = {}
job_status_budget = compute_job_status_budget(
@@ -3266,7 +3290,7 @@ def run_reconcile(
except Exception as e:
console_print(f"[error]Reporting `{CI_PASSED_CONTEXT}` for
`{LABEL_USE_PROJECT_CI}` failed: {e}[/]")
ci_passed_notes.append(
- f"reporting `{CI_PASSED_CONTEXT}` for `{LABEL_USE_PROJECT_CI}`
failed: {type(e).__name__}"
+ f"reporting `{CI_PASSED_CONTEXT}` for `{LABEL_USE_PROJECT_CI}`
failed: {describe_error(e)}"
)
attention.extend(bypass_finder.ambiguous)
attention.extend(job_reporter.delayed)
diff --git a/dev/breeze/tests/test_fork_ci_reconcile.py
b/dev/breeze/tests/test_fork_ci_reconcile.py
index f07374f87a5..17084130127 100644
--- a/dev/breeze/tests/test_fork_ci_reconcile.py
+++ b/dev/breeze/tests/test_fork_ci_reconcile.py
@@ -81,6 +81,7 @@ from airflow_breeze.utils.fork_ci_reconcile import (
compute_job_status_budget,
compute_projection,
count_jobs,
+ describe_error,
discover,
drop_reported,
fetch_fork_observations,
@@ -2410,7 +2411,7 @@ def
test_discover_keeps_going_when_the_project_ci_lookup_of_one_pull_request_fai
discover(client, configs, repo=REPO, now=NOW, unknown=unknown,
ci_passed_written=[])
- assert unknown == [(1, "`CI passed` not updated: GitHubError")]
+ assert unknown == [(1, "`CI passed` not updated: GitHubError: HTTP 502")]
assert client.rest_get.call_count == 2
@@ -4507,7 +4508,7 @@ def
test_run_reconcile_marks_older_pilot_pull_requests_and_reports_project_ci_ru
mock_reporter.assert_called_once_with(client, repo=REPO,
budget=job_status_budget, now=NOW, read_jobs={})
assert has_alerts is fails
if fails:
- assert "reporting `CI run started` for project CI failed: GitHubError"
in output.read_text()
+ assert "reporting `CI run started` for project CI failed: GitHubError:
HTTP 502" in output.read_text()
@pytest.mark.parametrize(
@@ -4985,7 +4986,7 @@ def
test_report_project_ci_passed_keeps_going_after_an_error(client):
attention: list[tuple[int, str]] = []
assert report_project_ci_passed(client, repo=REPO, attention=attention) ==
[2]
- assert attention == [(1, "`CI passed` not updated: GitHubError")]
+ assert attention == [(1, "`CI passed` not updated: GitHubError: HTTP 502")]
def _rest_for_project_ci_passed(runs, checks):
@@ -5321,7 +5322,8 @@ def
test_run_reconcile_reports_project_ci_passed_failure(mock_report_project_ci_
assert has_alerts is True
assert (
- "reporting `CI passed` for `use project ci` failed: GitHubError" in
(tmp_path / "s.json").read_text()
+ "reporting `CI passed` for `use project ci` failed: GitHubError: HTTP
502"
+ in (tmp_path / "s.json").read_text()
)
@@ -5519,3 +5521,43 @@ def
test_run_reconcile_compares_the_merge_the_fork_run_tested(mock_classify, moc
compare_base = mock_classify.call_args.kwargs["compare_base"]
assert compare_base.keywords["find_tested_merge"].func is read_tested_merge
+
+
[email protected](
+ ("error", "expected"),
+ [
+ pytest.param(
+ GitHubError("gh api graphql -f query=query { long } failed: HTTP
502: Bad Gateway\nmore"),
+ "GitHubError: HTTP 502: Bad Gateway",
+ id="gh-arguments-left-out",
+ ),
+ pytest.param(
+ GitHubError("GraphQL errors: [x]"), "GitHubError: GraphQL errors:
[x]", id="other-message"
+ ),
+ pytest.param(
+ GitHubError(
+ "gh api repos/apache/airflow/commits/"
+ + "a" * 40
+ + "/check-runs?check_name=CI+passed&per_page=100"
+ " --paginate still failing after 4 attempts: HTTP 502: Bad
Gateway"
+ ),
+ "GitHubError: still failing after 4 attempts: HTTP 502: Bad
Gateway",
+ id="retries-exhausted",
+ ),
+ pytest.param(
+ GitHubError("gh api graphql -f query=q still rate limited after 3
attempts"),
+ "GitHubError: still rate limited after 3 attempts",
+ id="rate-limited",
+ ),
+ pytest.param(
+ GitHubError("gh api repos/apache/airflow/pulls/1 timed out after
60s"),
+ "GitHubError: timed out after 60s",
+ id="timed-out",
+ ),
+ pytest.param(KeyError("sha"), "KeyError: 'sha'", id="other-exception"),
+ pytest.param(RuntimeError(), "RuntimeError", id="no-message"),
+ pytest.param(GitHubError("x" * 300), "GitHubError: " + "x" * 186 +
"…", id="shortened"),
+ ],
+)
+def test_describe_error(error, expected):
+ assert describe_error(error) == expected