This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 289e4a614b5a400fd9e64faee5dcaae75161b7f5 Author: Claus Ibsen <[email protected]> AuthorDate: Mon Nov 2 11:48:23 2026 +0100 ai-benchmark: seed files for an infra service, and checks on the files of its server The embedded SFTP server of camel infra serves the folder it was started in (rootDirMode is off, so its home is the working folder of the process). - infra_seed {service: folder under seed/}: the harness copies the folder to stepwise/<tag>/<name>/infra-<service>/ and starts the service in it, so every pass gets a fresh copy of the server's files, and the run keeps what the steps left there. - server_files {path or glob: regex} and server_not_files [path or glob]: checks on the files of that server. There is no fallback by name, as the files check has: a file of the same name in the project is not the file on the server. results.json lists the server's files per step. - The fast wait also waits for server_files (a receipt that a route uploads). - The checkpoint copies back the seed files that a route deleted or moved away, once the reference runs: after a failed step, and after a passed step whose files the reference replaced. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/README.md | 8 +++ ai-benchmark/agent_mcp_stepwise.py | 109 +++++++++++++++++++++++++++++++++++-- ai-benchmark/gen_stepwise.py | 4 ++ 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/ai-benchmark/README.md b/ai-benchmark/README.md index e88bc65..ab1e830 100644 --- a/ai-benchmark/README.md +++ b/ai-benchmark/README.md @@ -168,6 +168,12 @@ pass, which rewrites that example's steps file and resets its project, so each p after it. `contracts-openapi-client` calls the stock API in `seed/_peer-openapi-server` on port 8080. - `infra`: services started with `camel infra run` (the sql rung's Postgres); they stay up across the passes of a run and are stopped when the run is over. Docker must be running. +- `infra_seed`: `{service: folder under seed/}` for a service that serves files. The embedded SFTP server of + `camel infra run sftp` serves the folder it was started in, so the harness copies the folder to + `stepwise/<tag>/<name>/infra-<service>/` and starts the service there, a fresh copy in every pass + (`connect-sftp-transfer` puts its invoice batches from `seed/_sftp-server/` on the server). The checks + `server_files` and `server_not_files` read that folder, and the checkpoint copies back the seed files a step's route + deleted or moved away, after a failed step and after a passed one whose files it replaced. - `restart`: on a step that adds or changes Java, the harness restarts the app after the model's turn, as the README's step says; a reload does not compile a class. - `wait_seconds`: how long to wait after the reload before the log and files are checked. @@ -185,6 +191,8 @@ optional: | `file_not_regex` | the route file does not match | | `props_regex` | `application.properties` matches | | `files` | `{name: regex}`: another project file matches; found by name anywhere in the project if not at that path | +| `server_files` | `{path: regex}`: a file on the server of the `infra_seed` service matches; a path with `*` is a glob (`**/done/x.csv`), and any match counts | +| `server_not_files` | `[path]`: none of these files is on the server any more (a batch the route moved away) | | `log_regex` | a regex or a list of them; every one matches an INFO or WARN record (`min_log` times, default 1) | | `log_not_regex` | no record logged after this step's reload matches | | `interval_min` | the first two records matching `log_regex` are at least this many seconds apart | diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 18d10f4..4f1b6e7 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -187,19 +187,35 @@ FAST_NEG_FULL = os.environ.get("BENCH_FAST_NEG_FULL", "1") == "1" # a step that SERVER_INSTRUCTIONS = os.environ.get("BENCH_SERVER_INSTRUCTIONS") == "1" -def infra_start(services): +# the folder a seeded infra service runs in, by service: the embedded SFTP server of camel infra serves the files of +# the folder it was started in (its home), so the example's server files are put there before it starts, and the +# server_files checks read them there +INFRA_HOME = {} + + +def infra_start(services, seeds=None): """Start the services the example needs with `camel infra run <svc> --background` and wait until each answers. Stopped first, so every pass starts against an empty database. A service kept running between passes would carry its rows over, and then a step that changes a row passes on what the pass before it left rather than on the work of the model: step 3 sets C-207 to NL, and the next pass would find it already NL. Postgres takes about 80 s to come up, which is the price of that. The runner stops them when the whole run is over. + + seeds: {service: folder under seed/} for a service that serves files (sftp): the folder is copied to infra-<svc>/ + in the pass's output folder, and the service is started in it, so it serves a fresh copy in every pass. """ + seeds = seeds or {} for svc in services: subprocess.run(["camel", "infra", "stop", svc], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) for svc in services: - subprocess.run(["camel", "infra", "run", svc, "--background"], check=False, + cwd = HERE + if seeds.get(svc): + cwd = os.path.join(OUT, "infra-" + svc) + shutil.rmtree(cwd, ignore_errors=True) + shutil.copytree(os.path.join(HERE, "seed", seeds[svc]), cwd) + INFRA_HOME[svc] = cwd + subprocess.run(["camel", "infra", "run", svc, "--background"], check=False, cwd=cwd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) data, deadline = {}, time.time() + INFRA_TIMEOUT for svc in services: @@ -293,6 +309,63 @@ def file_matches(project, fname, rx): return bool(re.search(rx, project_file(project, fname) or "")) +def server_home(cfg): + """The folder of the example's seeded infra service (the files on its SFTP server), or None.""" + for svc in cfg.get("infra_seed") or {}: + if INFRA_HOME.get(svc): + return INFRA_HOME[svc] + return None + + +def server_paths(home, fname): + """The files on the server a check names: a path in the server's folder, or a glob (** for any folders, so a + batch the route moved to invoices/done and one it moved to done both count). Never the folder test-infra keeps + its own files in (target/).""" + import glob + if not home: + return [] + if "*" in fname: + found = glob.glob(os.path.join(home, fname), recursive=True) + else: + found = [os.path.join(home, fname)] + return [f for f in found if os.path.isfile(f) and not os.path.relpath(f, home).startswith("target" + os.sep)] + + +def server_file_matches(home, fname, rx): + """Whether a file on the server holds the regex; with a glob, any of the files it matches. No fallback by name, as + the project's files have: a file of the same name elsewhere is not the file on the server.""" + return any(re.search(rx, read(f) or "") for f in server_paths(home, fname)) + + +def server_listing(home): + """The files on the server, for the results (what a step left where).""" + out = [] + for root, dirs, files in os.walk(home or ""): + dirs[:] = [d for d in dirs if not (root == home and d == "target")] + out += [os.path.relpath(os.path.join(root, f), home) for f in files] + return sorted(out) + + +def restore_infra_seeds(cfg): + """Copies back the seed files of the seeded infra service that are gone from where the seed has them, such as the + invoice batches a step's route deleted or moved away from the server, so the next step starts from the files the + example starts with. Answers the files copied.""" + restored = [] + for svc, seed in (cfg.get("infra_seed") or {}).items(): + home, src = INFRA_HOME.get(svc), os.path.join(HERE, "seed", seed) + if not home or not os.path.isdir(src): + continue + for root, dirs, files in os.walk(src): + for f in files: + rel = os.path.relpath(os.path.join(root, f), src) + dst = os.path.join(home, rel) + if not os.path.exists(dst): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(os.path.join(root, f), dst) + restored.append(rel) + return restored + + def project_file(project, fname): """A file of the project by path, else by name anywhere in it (as score reads the files of a check).""" content = read(os.path.join(project, fname)) @@ -313,7 +386,10 @@ def wait_for_check(mcp, name, chk, cfg, project, before, reload_at): regexes = chk.get("log_regex") regexes = [] if regexes is None else (regexes if isinstance(regexes, list) else [regexes]) files = chk.get("files") or {} - if (not regexes and not files) or chk.get("min_errors") or (FAST_NEG_FULL and chk.get("log_not_regex")): + # the files a route writes on the server (a receipt it uploads) are waited for as the project's files are + server = chk.get("server_files") or {} + home = server_home(cfg) + if (not regexes and not files and not server) or chk.get("min_errors") or (FAST_NEG_FULL and chk.get("log_not_regex")): time.sleep(limit) return limit seen_msgs = before.get("log_records") or set() @@ -334,6 +410,7 @@ def wait_for_check(mcp, name, chk, cfg, project, before, reload_at): fresh = [msg(l) for l in log_lines(mcp, name, LOG_WINDOW) if fresh_ok(l)] logs_ok = all(sum(1 for m in fresh if re.search(rx, m, re.S)) >= need for rx in regexes) files_ok = all(file_matches(project, f, rx) for f, rx in files.items()) + files_ok = files_ok and all(server_file_matches(home, f, rx) for f, rx in server.items()) if logs_ok and files_ok: time.sleep(max(0.0, min(FAST_GRACE, limit - (time.time() - t0)))) return round(time.time() - t0, 1) @@ -450,6 +527,17 @@ def score(step, project, cfg, mcp, name, before): # a name with * is a glob if not file_matches(project, fname, rx): result["file_ok"] = False + # the files on the server of a seeded infra service: server_files {path or glob: regex} must be there and match, + # server_not_files [path or glob] must be gone (a batch the route moved away) + home = server_home(cfg) + if chk.get("server_files") or chk.get("server_not_files"): + for fname, rx in (chk.get("server_files") or {}).items(): + if not server_file_matches(home, fname, rx): + result["file_ok"] = False + if any(server_paths(home, fname) for fname in chk.get("server_not_files") or []): + result["file_ok"] = False + if home: + result["server_files"] = server_listing(home) lines = log_lines(mcp, name, LOG_WINDOW) def lvl(l): return (l.get("level") or "").upper() # a multi-line message (a pretty-printed body) comes as one record with a detail block: match on both @@ -571,9 +659,11 @@ def main(): # when the pass ends, so the next pass of the same example does not pay the startup again (about 80 s); the # runner stops them when the whole run is over t0 = time.time() - data = infra_start(cfg["infra"]) + data = infra_start(cfg["infra"], cfg.get("infra_seed")) print(f"camel infra run {' '.join(cfg['infra'])} -> {json.dumps(data)[:300]} ({time.time() - t0:.0f}s)", file=log, flush=True) + for svc, home in INFRA_HOME.items(): + print(f"{svc} serves {home}: {server_listing(home)}", file=log, flush=True) for svc, d in data.items(): if isinstance(d, dict) and d.get("error"): print(f"ABORT: {d['error']}", file=log, flush=True) @@ -785,6 +875,10 @@ def main(): os.makedirs(out, exist_ok=True) print(f"step{sid}: checkpoint emptied {d}", file=log, flush=True) differs = removed or restored or any(read(os.path.join(project, f)) != c for f, c in step["reference"].items()) + # the seed files of the example's server come back as well, once the reference runs (a route of the + # model's that deleted or moved the batches away is gone by then): after a failed step, and after a + # passed one whose files the reference replaced, as the next step's reference needs them + restore_server = bool(cfg.get("infra_seed")) and (not res["ok"] or differs) if FAST: # files that already equal the reference are not written again: no reload, nothing to wait for if differs: @@ -797,6 +891,10 @@ def main(): time.sleep(FAST_POLL) if any(reload_key(l) not in seen for l in log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)): break + back = restore_infra_seeds(cfg) if restore_server else [] + if back: + print(f"step{sid}: checkpoint restored server files {back}", file=log, flush=True) + if differs or back: res["settled"] = wait_quiet(mcp, name, FAST_QUIET, 6) else: apply_reference(project, cfg, step["reference"], mcp, name) @@ -810,6 +908,9 @@ def main(): time.sleep(2) if any(reload_key(l) not in seen for l in log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)): break + back = restore_infra_seeds(cfg) if restore_server else [] + if back: + print(f"step{sid}: checkpoint restored server files {back}", file=log, flush=True) time.sleep(6) if not res["ok"]: messages.append({"role": "user", "content": "I fixed that step myself; the files now contain the correct version. Continue with the next request."}) diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py index 849c56e..be3c80a 100644 --- a/ai-benchmark/gen_stepwise.py +++ b/ai-benchmark/gen_stepwise.py @@ -1155,6 +1155,10 @@ def main(): if ex.get("infra"): # services the example needs, started with `camel infra run` before the app (the sql example's postgres) cfg["infra"] = ex["infra"] + if ex.get("infra_seed"): + # {service: folder under seed/}: the files a service serves (the batches on the SFTP server), started in a + # fresh copy of the folder in every pass; the server_files checks read them there + cfg["infra_seed"] = ex["infra_seed"] if ex.get("clear_dirs"): # output folders the checkpoint of a failed step empties cfg["clear_dirs"] = ex["clear_dirs"]
