This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d2e60aefea2 Retry GitHub calls that drop the connection in the Spot
interruption notices (#75829)
d2e60aefea2 is described below
commit d2e60aefea255717df3aed0ccde4b4cb240d9c19
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Oct 30 16:56:44 2026 +0100
Retry GitHub calls that drop the connection in the Spot interruption
notices (#75829)
* Retry GitHub calls that drop the connection in the Spot interruption
notices
Only server errors counted as transient, so a connection GitHub dropped
mid-request (an HTTP/2 stream reset) was neither retried nor skipped like
one, and failed the scheduled run although the next run would do the same
work again.
Generated-by: Claude Opus 5
* Test each transient error pattern of the Spot notices' GitHub calls
Generated-by: Claude Opus 5
---
scripts/ci/spot_interruptions.py | 5 +++-
scripts/tests/ci/test_spot_interruptions.py | 36 ++++++++++++++++++++++++++++-
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/scripts/ci/spot_interruptions.py b/scripts/ci/spot_interruptions.py
index 08bb4d6da36..e78344fe074 100755
--- a/scripts/ci/spot_interruptions.py
+++ b/scripts/ci/spot_interruptions.py
@@ -80,7 +80,10 @@ COMMENT_AUTHOR = "github-actions[bot]"
# A job cancelled while its run was not, e.g. by its timeout, is a failure too.
FAILED_CONCLUSIONS = ("failure", "cancelled", "timed_out")
# GitHub answers with a server error while overloaded, and the same request
often succeeds later.
-TRANSIENT_ERROR = re.compile(r"\(HTTP 5\d\d\)")
+# Server errors, and connections GitHub dropped mid-request (an HTTP/2 stream
reset, for one).
+TRANSIENT_ERROR = re.compile(
+ r"\(HTTP 5\d\d\)|stream error|connection reset by peer|unexpected EOF|TLS
handshake timeout|i/o timeout"
+)
GH_API_ATTEMPTS = 3
GH_API_RETRY_DELAY_SECONDS = 5
diff --git a/scripts/tests/ci/test_spot_interruptions.py
b/scripts/tests/ci/test_spot_interruptions.py
index 29a6fc3fad3..d516890e658 100644
--- a/scripts/tests/ci/test_spot_interruptions.py
+++ b/scripts/tests/ci/test_spot_interruptions.py
@@ -297,9 +297,14 @@ def test_find_comment_ignores_comments_of_others(spot):
SERVER_ERROR = subprocess.CalledProcessError(1, ["gh"], stderr="gh: Server
Error (HTTP 502)\n")
NOT_FOUND_ERROR = subprocess.CalledProcessError(1, ["gh"], stderr="gh: Not
Found (HTTP 404)\n")
+STREAM_ERROR = subprocess.CalledProcessError(
+ 1, ["gh"], stderr="stream error: stream ID 3; CANCEL; received from peer\n"
+)
[email protected](("error", "expected_ok"), [(SERVER_ERROR, True),
(NOT_FOUND_ERROR, False)])
[email protected](
+ ("error", "expected_ok"), [(SERVER_ERROR, True), (STREAM_ERROR, True),
(NOT_FOUND_ERROR, False)]
+)
def test_run_pr_comments_carries_on_past_a_failing_run(spot, capsys, error,
expected_ok):
runs = [_run(branch="a"), _run(branch="b")]
@@ -329,6 +334,7 @@ def test_run_pr_comments_when_listing_the_runs_fails(spot,
error, expected_ok):
[
pytest.param(("repos/x",), [SERVER_ERROR, SERVER_ERROR], 3, None,
id="retries-server-errors"),
pytest.param(("repos/x",), [SERVER_ERROR] * 3, 3, SERVER_ERROR,
id="gives-up"),
+ pytest.param(("repos/x",), [STREAM_ERROR], 2, None,
id="retries-dropped-connections"),
pytest.param(("repos/x",), [NOT_FOUND_ERROR], 1, NOT_FOUND_ERROR,
id="no-retry-on-client-error"),
pytest.param(("-X", "POST", "repos/x"), [SERVER_ERROR], 1,
SERVER_ERROR, id="no-retry-on-post"),
pytest.param(("-X", "PATCH", "repos/x"), [SERVER_ERROR], 2, None,
id="retries-patch"),
@@ -350,6 +356,34 @@ def test_gh_api_retries_server_errors(
assert mock_run.call_count == expected_calls
[email protected](
+ ("stderr", "expected"),
+ [
+ pytest.param("gh: Server Error (HTTP 502)\n", True, id="server-error"),
+ pytest.param("stream error: stream ID 3; CANCEL; received from
peer\n", True, id="stream-reset"),
+ pytest.param(
+ 'Get "https://api.github.com/repos/x": read tcp
10.1.0.4:52144->140.82.112.6:443: '
+ "read: connection reset by peer\n",
+ True,
+ id="connection-reset",
+ ),
+ pytest.param('Get "https://api.github.com/repos/x": unexpected EOF\n',
True, id="unexpected-eof"),
+ pytest.param(
+ 'Get "https://api.github.com/repos/x": net/http: TLS handshake
timeout\n', True, id="tls-timeout"
+ ),
+ pytest.param(
+ 'Get "https://api.github.com/repos/x": dial tcp 140.82.112.6:443:
i/o timeout\n',
+ True,
+ id="dial-timeout",
+ ),
+ pytest.param("gh: Not Found (HTTP 404)\n", False, id="client-error"),
+ pytest.param(None, False, id="no-stderr"),
+ ],
+)
+def test_is_transient(spot, stderr, expected):
+ assert spot.is_transient(subprocess.CalledProcessError(1, ["gh"],
stderr=stderr)) is expected
+
+
def test_run_slack_writes_outputs(spot, monkeypatch, tmp_path):
output = tmp_path / "output"
monkeypatch.setenv("GITHUB_OUTPUT", str(output))
