This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d1bdea5093 [fix](ci) Resume reviews with Astra after five capacity
failures (#69953)
5d1bdea5093 is described below
commit 5d1bdea509364e1a8b245c2482a38bf14325b994
Author: shuke <[email protected]>
AuthorDate: Mon Nov 2 14:12:53 2026 +0800
[fix](ci) Resume reviews with Astra after five capacity failures (#69953)
### What problem does this PR solve?
Reviews can exhaust all seven attempts while the selected model remains
at capacity. After five capacity failures, this change resumes the same
review session with `gpt-6-astra` for the remaining two attempts,
preserving the existing five-minute backoff, shared deadline, target
checks, and final-review verification.
The workflow opts in through `--capacity-fallback-model`. Account/model
admission fallback remains separate. Recovery instructions preserve
completed findings and the cumulative round count, and direct unfinished
child-agent work to the selected model.
Scope: three files; 17 production lines and 91 test lines added.
---
.github/scripts/run_review_with_resume.py | 16 +++++
.github/scripts/test_run_review_with_resume.py | 91 ++++++++++++++++++++++++++
.github/workflows/code-review-runner.yml | 1 +
3 files changed, 108 insertions(+)
diff --git a/.github/scripts/run_review_with_resume.py
b/.github/scripts/run_review_with_resume.py
index e02c0d9dd78..484deeba4e2 100644
--- a/.github/scripts/run_review_with_resume.py
+++ b/.github/scripts/run_review_with_resume.py
@@ -38,6 +38,7 @@ from submit_review import RUN_FILE, RESULT_FILE,
SUBMISSION_FILE, verify_complet
# Six same-session capacity retries, each after a fixed five-minute wait.
RETRY_DELAYS = (300,) * 6
+CAPACITY_FALLBACK_AFTER = 5
CAPACITY_MESSAGE = "Selected model is at capacity. Please try a different
model."
UNSUPPORTED_CHATGPT_MODEL = "model is not supported when using Codex with a
ChatGPT account."
PROCESS_EXIT_GRACE_SECONDS = 5
@@ -438,6 +439,7 @@ def run_review(args, reaper=None):
thread_id = None
model = args.model
fallback_model = getattr(args, "fallback_model", None)
+ capacity_fallback_model = getattr(args, "capacity_fallback_model", None)
capacity_retry = 0
attempt_number = 0
@@ -608,6 +610,19 @@ def run_review(args, reaper=None):
watchdog.wait(delay)
check_resume_target(args, started_at, remaining)
capacity_retry += 1
+ if capacity_retry == CAPACITY_FALLBACK_AFTER and
capacity_fallback_model:
+ print(
+ f"Switching capacity recovery from {model} to
{capacity_fallback_model} "
+ f"after {CAPACITY_FALLBACK_AFTER} capacity failures;
keeping session {thread_id}",
+ file=sys.stderr, flush=True,
+ )
+ model = capacity_fallback_model
+ goal_prompt += (
+ f"\n\nCapacity recovery switched to {model}. Use this
model for remaining "
+ "review work, including child agents. If an unfinished
child cannot resume "
+ "with this model, replace only its unfinished work using
its recorded ledger "
+ "section; preserve completed findings and the cumulative
round count."
+ )
except ReviewSuperseded as exc:
result = {
"state": "superseded", "repository": args.repository,
@@ -641,6 +656,7 @@ def main():
parser.add_argument("--base-sha", required=True)
parser.add_argument("--model", required=True)
parser.add_argument("--fallback-model")
+ parser.add_argument("--capacity-fallback-model", help="Resume with this
model after five capacity failures")
parser.add_argument("--effort", required=True)
# The workflow owns the total timeout and deducts setup/finalization time.
parser.add_argument("--budget-seconds", type=int, required=True)
diff --git a/.github/scripts/test_run_review_with_resume.py
b/.github/scripts/test_run_review_with_resume.py
index 313a740129e..97018bff25c 100644
--- a/.github/scripts/test_run_review_with_resume.py
+++ b/.github/scripts/test_run_review_with_resume.py
@@ -86,6 +86,7 @@ class ResumeReviewTest(unittest.TestCase):
base_sha="b" * 40,
model="gpt-5.6-sol",
fallback_model=None,
+ capacity_fallback_model="gpt-6-astra",
effort="xhigh",
budget_seconds=1000,
)
@@ -425,6 +426,91 @@ class ResumeReviewTest(unittest.TestCase):
self.assertEqual("done", (self.context /
"codex-final-message.txt").read_text())
self.target_check.assert_called_once()
+ def test_capacity_fallback_resumes_same_session_after_five_failures(self):
+ self.args.model = "gpt-6.1-sol"
+ self.args.budget_seconds = 3600
+ before = self.ledger.read_text()
+ self.assertEqual(0, self.execute(
+ [{"events": [thread_event(), failed()], "elapsed": 5}] * 5
+ + [{"events": [thread_event(), completed()], "status": 0}]
+ ))
+ self.assertEqual(
+ ["gpt-6.1-sol"] * 5 + ["gpt-6-astra"],
+ [c[c.index("--model") + 1] for c in self.commands],
+ )
+ self.assertEqual([3600, 3295, 2990, 2685, 2380, 2075], self.timeouts)
+ self.assertEqual([300] * 5, self.sleeps)
+ for command in self.commands[1:]:
+ self.assertEqual(["resume", THREAD], command[-3:-1])
+ self.assertIn("model_reasoning_effort=xhigh", command)
+ self.assertNotIn("Capacity recovery switched", self.commands[4][-1])
+ self.assertIn("Capacity recovery switched to gpt-6-astra",
self.commands[5][-1])
+ self.assertIn("including child agents", self.commands[5][-1])
+ self.assertIn("cumulative round count", self.commands[5][-1])
+ self.assertEqual(before, self.ledger.read_text())
+ self.assertEqual("gpt-6-astra\n", (self.context /
"codex-review-model.txt").read_text())
+ self.verification.assert_called_once()
+
+ def test_capacity_fallback_is_optional(self):
+ self.args.capacity_fallback_model = None
+ self.args.budget_seconds = 3600
+ self.assertEqual(1, self.execute([{"events": [thread_event(),
failed()]}] * 7))
+ self.assertEqual(
+ [self.args.model] * 7,
+ [c[c.index("--model") + 1] for c in self.commands],
+ )
+
+ def test_fifth_attempt_non_capacity_failure_does_not_switch(self):
+ self.args.budget_seconds = 3600
+ self.assertEqual(1, self.execute(
+ [{"events": [thread_event(), failed()]}] * 4
+ + [{"events": [thread_event(), failed("HTTP 422")]}]
+ ))
+ self.assertEqual(5, len(self.commands))
+ self.assertEqual("HTTP 422", self.last_error())
+ self.assertEqual(self.args.model + "\n", (self.context /
"codex-review-model.txt").read_text())
+
+ def test_capacity_fallback_preserves_budget_and_target_checks(self):
+ self.args.budget_seconds = 1500
+ self.assertEqual(1, self.execute([{"events": [thread_event(),
failed()]}] * 5))
+ self.assertEqual(5, len(self.commands))
+ self.assertEqual([300] * 4, self.sleeps)
+ self.assertIn("Insufficient shared review budget", self.last_error())
+ self.assertEqual(self.args.model + "\n", (self.context /
"codex-review-model.txt").read_text())
+
+ def test_target_change_before_capacity_switch_stops_same_review(self):
+ self.args.budget_seconds = 3600
+ self.target_check.side_effect = [None] * 4 +
[runner.ReviewSuperseded("c" * 40)]
+ self.assertEqual(75, self.execute([{"events": [thread_event(),
failed()]}] * 5))
+ self.assertEqual(5, len(self.commands))
+ self.assertEqual(self.args.model + "\n", (self.context /
"codex-review-model.txt").read_text())
+ self.assert_superseded()
+
+ def test_submission_at_fifth_capacity_is_verified_without_switch(self):
+ self.args.budget_seconds = 3600
+ read_events = runner.read_events
+ def submitted(path):
+ if path.name == "5.jsonl":
+ (self.context / runner.SUBMISSION_FILE).write_text("{}")
+ return read_events(path)
+ with mock.patch.object(runner, "read_events", side_effect=submitted):
+ self.assertEqual(0, self.execute([{"events": [thread_event(),
failed()]}] * 5))
+ self.assertEqual(5, len(self.commands))
+ self.assertEqual([300] * 4, self.sleeps)
+ self.verification.assert_called_once()
+ self.assertTrue(json.loads((self.context /
runner.RESULT_FILE).read_text())["recovered_after_capacity"])
+
+ def
test_unsupported_capacity_fallback_does_not_restart_or_switch_back(self):
+ self.args.budget_seconds = 3600
+ rejection = "The 'gpt-6-astra' " + runner.UNSUPPORTED_CHATGPT_MODEL
+ self.assertEqual(1, self.execute(
+ [{"events": [thread_event(), failed()]}] * 5
+ + [{"events": [thread_event(), failed(rejection)]}]
+ ))
+ self.assertEqual(6, len(self.commands))
+ self.assertEqual(["resume", THREAD], self.commands[-1][-3:-1])
+ self.assertEqual(rejection, self.last_error())
+
def test_zero_exit_capacity_can_complete_on_the_sixth_resume(self):
self.args.budget_seconds = 3600
self.assertEqual(
@@ -579,6 +665,10 @@ class ResumeReviewTest(unittest.TestCase):
self.assertEqual(1, self.execute([{"events": [thread_event(),
failed()]}] * 7))
self.assertEqual([300] * 6, self.sleeps)
self.assertEqual(7, len(self.commands))
+ self.assertEqual(
+ [self.args.model] * 5 + ["gpt-6-astra"] * 2,
+ [c[c.index("--model") + 1] for c in self.commands],
+ )
self.assertEqual(runner.CAPACITY_MESSAGE, self.last_error())
def
test_no_retry_for_auth_usage_or_generic_errors_even_before_session_starts(self):
@@ -1056,6 +1146,7 @@ HELPER
check=True,
)
args = json.loads(result.stdout)
+ self.assertEqual("gpt-6-astra",
args[args.index("--capacity-fallback-model") + 1])
actual = int(args[args.index("--budget-seconds") + 1])
self.assertLessEqual(actual, expected)
self.assertGreaterEqual(actual, expected - 2)
diff --git a/.github/workflows/code-review-runner.yml
b/.github/workflows/code-review-runner.yml
index dcd6f23f23f..cc763b09371 100644
--- a/.github/workflows/code-review-runner.yml
+++ b/.github/workflows/code-review-runner.yml
@@ -917,6 +917,7 @@ jobs:
--base-sha "$BASE_SHA" \
--model "gpt-6.1-sol" \
--fallback-model "gpt-6-sol" \
+ --capacity-fallback-model "gpt-6-astra" \
--effort xhigh \
--budget-seconds "$budget_seconds"
status=$?
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
