This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch ci-sync-v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a790d1eb76d1d7e0612efac4954f7182a19c3547 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Oct 27 05:00:11 2026 +0100 Retry GitHub calls that fail to connect in the fork CI monitor (#75701) A monitor run failed to report job statuses for a PR because a POST to the statuses API ended in a TLS handshake timeout. Network failures are as transient as the server errors already retried, and one that happens before the request reaches GitHub cannot have taken effect, so it is safe to repeat even for calls such as creating a comment. Generated-by: Claude Opus 5 (cherry picked from commit 2ae00f1606ab4fefb12952270826efc2e07b8697) --- .../src/airflow_breeze/utils/fork_ci_github.py | 25 +++++++++++++++---- dev/breeze/tests/test_fork_ci_github.py | 29 +++++++++++++++++++++- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py index df00485a2b1..721ecc7873d 100644 --- a/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py +++ b/dev/breeze/src/airflow_breeze/utils/fork_ci_github.py @@ -33,8 +33,11 @@ from airflow_breeze.utils.fork_ci_messages import STATUS_CONTEXT RATE_LIMIT_MARKERS = ("rate limit", "secondary rate", "abuse detection") MAX_ATTEMPTS = 4 # GitHub answers with these while overloaded, and GraphQL searches that take too long end in HTTP 504; -# the same call usually succeeds shortly after. -SERVER_ERROR = re.compile(r"\bHTTP 5\d\d\b") +# the same call usually succeeds shortly after. A connection dropped mid-request is as transient, and as +# unclear about whether the request took effect. +SERVER_ERROR = re.compile(r"\bHTTP 5\d\d\b|connection reset by peer|unexpected EOF") +# Network failures before the request reached GitHub, so repeating it is safe for any call. +CONNECTION_ERROR = re.compile(r"TLS handshake timeout|dial tcp") SERVER_ERROR_WAIT_SECONDS = 10 # A hung `gh` would otherwise hold the job until its timeout-minutes. GH_TIMEOUT_SECONDS = 120 @@ -64,7 +67,8 @@ class GitHubClient: graphql`` exits non-zero when some fields of the response errored, with the data GitHub could resolve still on stdout. :param retry_server_errors: repeat a call that failed with a server error. Off for a call that - may have taken effect anyway and must not be repeated, such as creating a comment. + may have taken effect anyway and must not be repeated, such as creating a comment. A call + that failed to connect is repeated either way. """ payload = json.dumps(body) if body is not None else None rate_limited = False @@ -84,10 +88,21 @@ class GitHubClient: if result.returncode == 0: return json.loads(result.stdout) if result.stdout.strip() else None rate_limited = any(marker in result.stderr.lower() for marker in RATE_LIMIT_MARKERS) - if rate_limited or (retry_server_errors and SERVER_ERROR.search(result.stderr)): + connection_error = bool(CONNECTION_ERROR.search(result.stderr)) + if ( + rate_limited + or connection_error + or (retry_server_errors and SERVER_ERROR.search(result.stderr)) + ): if attempt == MAX_ATTEMPTS - 1: break - reason = "rate limit" if rate_limited else "server error" + reason = ( + "rate limit" + if rate_limited + else "connection error" + if connection_error + else "server error" + ) wait = (30 if rate_limited else SERVER_ERROR_WAIT_SECONDS) * 2**attempt console_print(f"[warning]GitHub {reason}, retrying in {wait}s[/]") time.sleep(wait) diff --git a/dev/breeze/tests/test_fork_ci_github.py b/dev/breeze/tests/test_fork_ci_github.py index 1bd85bcb749..9f6505e2954 100644 --- a/dev/breeze/tests/test_fork_ci_github.py +++ b/dev/breeze/tests/test_fork_ci_github.py @@ -53,7 +53,17 @@ def test_gives_up_after_the_last_attempt_without_waiting(mock_run, mock_sleep): assert [c.args[0] for c in mock_sleep.call_args_list] == [30, 60, 120] [email protected]("stderr", ["gh: HTTP 504", "gh: Server Error (HTTP 502)"]) [email protected]( + "stderr", + [ + "gh: HTTP 504", + "gh: Server Error (HTTP 502)", + 'Post "https://api.github.com/graphql": read tcp 10.1.0.4:51234->140.82.112.6:443: read: connection reset by peer', + 'Post "https://api.github.com/graphql": unexpected EOF', + 'Post "https://api.github.com/graphql": net/http: TLS handshake timeout', + 'Post "https://api.github.com/graphql": dial tcp 140.82.112.6:443: i/o timeout', + ], +) @mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) @mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) def test_retries_on_server_errors(mock_run, mock_sleep, stderr): @@ -90,6 +100,23 @@ def test_does_not_repeat_creating_a_comment_after_a_server_error(mock_run, mock_ mock_sleep.assert_not_called() [email protected]( + "stderr", + [ + 'Post "https://api.github.com/repos/x/y/issues/10/comments": net/http: TLS handshake timeout', + 'Post "https://api.github.com/repos/x/y/issues/10/comments": dial tcp: lookup api.github.com: no such host', + ], +) [email protected]("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True) [email protected]("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True) +def test_repeats_creating_a_comment_that_did_not_connect(mock_run, mock_sleep, stderr): + mock_run.side_effect = [_completed(1, stderr=stderr), _completed(0, stdout="{}")] + + GitHubClient().create_comment(10, "body") + assert mock_run.call_count == 2 + mock_sleep.assert_called_once_with(10) + + @mock.patch( "airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True,
