This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit ccd3f0e58ac5201639a0847dfc2e592787fcef52 Author: Claus Ibsen <[email protected]> AuthorDate: Mon Nov 2 12:24:35 2026 +0100 ai-benchmark: checkpoint_requests, HTTP requests that reset the state of a service when the checkpoint resets a step The checkpoint puts the reference files in place, but a service keeps what an earlier try put in it. In the model run of connect-aws-s3-sqs, the model's first tries in step 2 sent messages to the SQS queue that were not JSON. Its step 3 consumer was right, and still failed on them, and so did step 4, when the messages came back after their visibility timeout. An example may now list checkpoint_requests (method, url, headers, body, note). The checkpoint sends them after a failed step, and after a passed one whose files it replaced, once the reference runs, as it copies back the seed files of a server. Each request is sent once, and its answer is only logged: a queue that does not exist yet answers 400. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/README.md | 4 ++++ ai-benchmark/agent_mcp_stepwise.py | 28 ++++++++++++++++++++++++++-- ai-benchmark/gen_stepwise.py | 3 +++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ai-benchmark/README.md b/ai-benchmark/README.md index 95030b0..9da206a 100644 --- a/ai-benchmark/README.md +++ b/ai-benchmark/README.md @@ -175,6 +175,10 @@ pass, which rewrites that example's steps file and resets its project, so each p (`connect-sftp-transfer` puts its invoice batches from `seed/_sftp-server/` on the server). The checks `server_files` and `server_not_files` read that folder, and the checkpoint copies back the seed files a step's route deleted or moved away, after a failed step and after a passed one whose files it replaced. +- `checkpoint_requests`: HTTP requests (`method`, `url`, `headers`, `body`, `note`) that the checkpoint sends after a + failed step and after a passed one whose files it replaced, once the reference runs: the state of a service that + the files do not hold. `connect-aws-s3-sqs` purges its SQS queue, as messages that a model's earlier try could not + write as JSON made the next steps' consumer fail. - `restart`: on a step that adds or changes Java, the harness restarts the app after the model's turn, as the README's step says; a reload does not compile a class. - `wait_seconds`: how long to wait after the reload before the log and files are checked. diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 727f688..8c35a4c 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -369,6 +369,26 @@ def restore_infra_seeds(cfg): return restored +def checkpoint_requests(cfg, sid, log): + """Sends the example's checkpoint_requests: HTTP requests that reset the state of a service, which the files do + not hold, once the reference runs (the SQS queue that a model's earlier try filled with messages the next step's + consumer cannot read: the model run awsm1 of connect-aws-s3-sqs failed steps 3 and 4 on them). Each is sent once, and its answer is + only logged (a queue that does not exist yet answers 400). Answers how many were sent.""" + for rq in cfg.get("checkpoint_requests") or []: + try: + data = rq["body"].encode() if rq.get("body") is not None else None + req = urllib.request.Request(rq["url"], data=data, method=rq.get("method", "POST"), headers=rq.get("headers") or {}) + with urllib.request.urlopen(req, timeout=8) as r: + status = r.status + except urllib.error.HTTPError as e: + status = e.code + except Exception as e: + status = str(e)[:80] + print(f"step{sid}: checkpoint request {rq.get('method', 'POST')} {rq['url']} {rq.get('note', '')} -> {status}", + file=log, flush=True) + return len(cfg.get("checkpoint_requests") or []) + + def project_file(project, fname): """A file of the project by path, else by name anywhere in it (as score reads the files of a check).""" content = read(os.path.join(project, fname)) @@ -881,7 +901,8 @@ def main(): # the seed files of the example's server come back as well, once the reference runs (a route of the # model's that deleted or moved the batches away is gone by then): after a failed step, and after a # passed one whose files the reference replaced, as the next step's reference needs them - restore_server = bool(cfg.get("infra_seed")) and (not res["ok"] or differs) + reset = not res["ok"] or differs + restore_server = bool(cfg.get("infra_seed")) and reset if FAST: # files that already equal the reference are not written again: no reload, nothing to wait for if differs: @@ -897,7 +918,8 @@ def main(): back = restore_infra_seeds(cfg) if restore_server else [] if back: print(f"step{sid}: checkpoint restored server files {back}", file=log, flush=True) - if differs or back: + sent = checkpoint_requests(cfg, sid, log) if reset else 0 + if differs or back or sent: res["settled"] = wait_quiet(mcp, name, FAST_QUIET, 6) else: apply_reference(project, cfg, step["reference"], mcp, name) @@ -914,6 +936,8 @@ def main(): back = restore_infra_seeds(cfg) if restore_server else [] if back: print(f"step{sid}: checkpoint restored server files {back}", file=log, flush=True) + if reset: + checkpoint_requests(cfg, sid, log) time.sleep(6) if not res["ok"]: messages.append({"role": "user", "content": "I fixed that step myself; the files now contain the correct version. Continue with the next request."}) diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py index 04ee906..4aec1fe 100644 --- a/ai-benchmark/gen_stepwise.py +++ b/ai-benchmark/gen_stepwise.py @@ -1228,6 +1228,9 @@ def main(): if ex.get("clear_dirs"): # output folders the checkpoint of a failed step empties cfg["clear_dirs"] = ex["clear_dirs"] + if ex.get("checkpoint_requests"): + # HTTP requests the checkpoint sends when it resets a step: the state of a service the files do not hold + cfg["checkpoint_requests"] = ex["checkpoint_requests"] if ex.get("peer"): cfg["peer"] = {"project": ex["peer"]["project"], "name": ex["peer"]["name"]} pd = os.path.join(HERE, ex["peer"]["project"])
