This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit bd98ebf52e3765c1e87b20f74ab78141a85eb18c Author: Claus Ibsen <[email protected]> AuthorDate: Thu Nov 5 11:06:55 2026 +0100 ai-benchmark: an error after the last write counts only if its exchange began after that write r27n jms s3 had the final state right in 6 of 7 runs, but r27n-2, -3 and -6 lost "last version right" to 1 error each. It was the request-reply of reserve-order, for an exchange that began 6 to 10 s before the last write, on the version before it. The reload of that write cut the request off. The error came from camel_get_errors only (a cut request logs no ERROR line), and its record reached the error file after the write's snapshot: the app writes that file about once a second. The errors after the last write now leave out a record whose exchange began (timestamp - elapsed) before the last write that the app reloaded. An unchanged edit, or a write the app did not reload, starts no new version. A record without these times still counts, and so do the ERROR lines of the log. results.json records the count left out as errors_older_version. A re-score of the r26n, r26l, r27n and r27l traces changes only these 3 step runs (r27n last version right 164 -> 167). The rule only leaves errors out, and no other step run had the final state right but not the last version. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/agent_mcp_stepwise.py | 39 +++++++++++++++++++++++++++++++++++++- ai-benchmark/summarize_stepwise.py | 3 ++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 86e6741..0c53dfa 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -174,6 +174,27 @@ def error_key(l): return (l.get("time") or l.get("timestamp") or "") + "|" + (l.get("message") or l.get("msg") or "")[:120] +def reloaded(out): + """Whether a write's answer says the app reloaded what it wrote: not an unchanged edit, a failed reload, or a file + the app does not reload.""" + try: + return (json.loads(out).get("reload") or {}).get("status") == "reloaded" + except Exception: + return False + + +def began_before(e, t): + """Whether the exchange of an error record began before t (epoch seconds), with half a second to spare as for the + step's start: the record's time (epoch millis, when it failed) less how long the exchange had run. False when the + record does not tell.""" + if t is None or not isinstance(e, dict): + return False + ts, elapsed = e.get("timestamp"), e.get("elapsed") + if not isinstance(ts, (int, float)) or not isinstance(elapsed, (int, float)): + return False + return ts - elapsed < (t - 0.5) * 1000 + + INFRA_TIMEOUT = int(os.environ.get("BENCH_INFRA_TIMEOUT", "300")) # seconds to wait for `camel infra` services # BENCH_FAST=1: wait for what a step checks instead of fixed sleeps (Oct 14 2026: 22 of a run's 55 minutes were # fixed sleeps, while dev mode now reloads within ~2 s of a write, CAMEL-25032) @@ -751,14 +772,26 @@ def score(step, project, cfg, mcp, name, before, log=None): # ok_lastwrite: the end state is right and the model's last version logged no errors; the errors of attempts it # fixed on the way do not count (they do in ok). Without a write in the step it is the same as ok last = before.get("last_write") + older = 0 if last is not None: after = sum(1 for l in lines if isinstance(l, dict) and lvl(l) == "ERROR" and error_key(l) not in last[0] and error_key(l) not in seen) if isinstance(data, dict): - after += max(0, len(data.get("errors", []) or []) - max(last[1], before.get("error_count", 0))) + errs = data.get("errors", []) or [] + # the records that came after the last write are the newest ones (by uid; the file lists them newest first). + # A record whose exchange began before the last write that the app reloaded is from the version before it, + # even when it failed after it: a request in flight that the reload cut off (r27n jms s3, 3 runs). Its record + # reached the error file after the write's snapshot, as the app writes that file about once a second. A + # record that does not tell when its exchange began counts, and so do the ERROR records of the log + newest = sorted(errs, key=lambda e: (e.get("uid") or 0) if isinstance(e, dict) else 0, reverse=True) + newest = newest[:max(0, len(errs) - max(last[1], before.get("error_count", 0)))] + older = sum(1 for e in newest if began_before(e, before.get("last_write_at"))) + after += len(newest) - older else: after = result["errors"] result["errors_after_last_write"] = after + # how many errors after the last write were left out, as their exchange began before it + result["errors_older_version"] = older lastwrite_fine = after == 0 or bool(chk.get("errors_ok")) if chk.get("min_errors") and result["errors"] < chk["min_errors"]: lastwrite_fine = False @@ -927,6 +960,10 @@ def main(): refused += 1 elif fn["name"] in ("camel_write_file", "camel_edit_file"): before["last_write"] = error_snapshot(mcp, name) + # when the app's current version began (ok_lastwrite: an exchange that began before it ran on + # the version before). Only a write that the app reloaded starts a version + if reloaded(out): + before["last_write_at"] = t_call chars = len(out) out = out[:CATALOG_RESULT_CAP if fn["name"] == "camel_catalog_doc" else TOOL_RESULT_CAP] trace.write(json.dumps({"tool": fn["name"], "ms": ms, "args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()}, "result": out[:TRACE_RESULT_CHARS], "chars": chars, "shown": len(out)}) + "\n"); trace.flush() diff --git a/ai-benchmark/summarize_stepwise.py b/ai-benchmark/summarize_stepwise.py index 171805a..d376d44 100644 --- a/ai-benchmark/summarize_stepwise.py +++ b/ai-benchmark/summarize_stepwise.py @@ -4,7 +4,8 @@ A step passes (ok) when its files, log and probes are right and it logged no new errors; final state right (ok_final) leaves the errors out: a model that tests its own app (the HTTP request tool) causes errors on the way to a right answer, which ok counts against it. Last version right (ok_lastwrite) is in between: the final state is right and no -errors were logged after the model's last accepted write; runs before it was recorded count it as ok.""" +errors were logged after the model's last accepted write, but for those of exchanges that began before that write (since +Nov 5 2026: they ran on the version before); runs before it was recorded count it as ok.""" import json, os, sys tag = sys.argv[1]; k = int(sys.argv[2]) if len(sys.argv) > 2 else 1 tags = [f"{tag}-{i}" for i in range(1, k + 1)] if k > 1 else [tag]
