This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 53762a2928a9356be61babe6ee88eff0c60ae2ad Author: Claus Ibsen <[email protected]> AuthorDate: Mon Nov 2 11:48:12 2026 +0100 ai-benchmark: BENCH_NEGATIVE=1, a pass that writes nothing, so each step is scored on the reference of the step before it A step's check must not pass on what the step before it left: such a check passes without the work that the step asks for. The negative pass checks this for every step at once. It asks no model and writes no reference files, so each step is scored on the reference that the checkpoint of the step before it put in place (step 1 on the start files). Every step must fail. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/README.md | 3 +++ ai-benchmark/agent_mcp_stepwise.py | 10 ++++++++-- ai-benchmark/run-stepwise-ladder.sh | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ai-benchmark/README.md b/ai-benchmark/README.md index 7c9766f..e88bc65 100644 --- a/ai-benchmark/README.md +++ b/ai-benchmark/README.md @@ -144,6 +144,7 @@ scores each step before sending the next. ./start-server.sh # set CAMEL_MCP_JAR to measure a locally built camel-jbang-mcp python3 gen_stepwise.py # writes steps-ladder/<name>.json and stepwise-ladder/<name>/ for every example BENCH_REFERENCE=1 ./run-stepwise-ladder.sh ref # the reference pass: must pass every step before a model run +BENCH_NEGATIVE=1 ./run-stepwise-ladder.sh neg # the negative pass: nothing written, so every step must fail ./run-stepwise-ladder.sh s1 5 # five runs: stepwise/s1-1 .. s1-5, then the summary table ./run-http.sh h1 3 # only the four HTTP rungs: stock-api, http-client, openapi-server/-client BENCH_ONLY="transform-xslt route-aggregator" ./run-stepwise-ladder.sh x 1 # any subset, names as in steps-ladder/ @@ -222,5 +223,7 @@ calls, refused writes, seconds, tokens, the model's closing answer), `run.log` ( and a reload that logs writes again. Output belongs under `stepwise/`, which is outside the projects. - Run the reference pass after any change to `gen_stepwise.py`, a seed, or Camel: a reference that fails is a harness or Camel bug, not a model result. +- Run the negative pass for a new or changed example: each step is scored on the reference of the step before it + (the start files for step 1), and a step that passes there has a check that does not need the step's work. - Do not share one MCP server between two runs, a reference pass included: the step logs and error counts of one get mixed into the other. diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 3c931f0..18d10f4 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -37,6 +37,10 @@ NUM_PREDICT = int(os.environ.get("BENCH_NUM_PREDICT", "8192")) LOG_WINDOW = int(os.environ.get("BENCH_LOG_WINDOW", "400")) # round 2: BENCH_REFERENCE=1 skips the model and applies each step's reference files instead, to check the steps file itself REFERENCE = os.environ.get("BENCH_REFERENCE") == "1" +# BENCH_NEGATIVE=1, the negative pass: skips the model and writes nothing, so each step is scored on what the step before +# it left (the checkpoint puts that step's reference in place, the start files for step 1). Every step must fail: a check +# that passes here passes without the work the step asks for +NEGATIVE = os.environ.get("BENCH_NEGATIVE") == "1" # CAMEL-24834: before each step, ask camel_runtime_tool_groups what the app has and offer its groups' tools and guidance, # as the AI panel of the camel-jbang views does for a local model (the tool list changes only when the fingerprint does) TOOL_GROUPS = os.environ.get("BENCH_TOOL_GROUPS") == "1" @@ -626,7 +630,7 @@ def main(): try: for step in cfg["steps"]: sid = step["id"] - if TOOL_GROUPS and not REFERENCE and not BARE: + if TOOL_GROUPS and not REFERENCE and not NEGATIVE and not BARE: data, raw = jcall(mcp, "camel_runtime_tool_groups", {"nameOrPid": name}) if isinstance(data, dict) and data.get("fingerprint") != groups_fp: groups_fp = data.get("fingerprint") @@ -650,7 +654,9 @@ def main(): # the reference files stand in for the model's edits; the checks then score the steps file itself writes += apply_reference(project, cfg, step.get("reference") or {}, mcp, name, before) answer = "(reference)" - while not REFERENCE: + elif NEGATIVE: + answer = "(negative pass: nothing written)" + while not REFERENCE and not NEGATIVE: try: try: data, secs = ollama_chat(messages, tools) diff --git a/ai-benchmark/run-stepwise-ladder.sh b/ai-benchmark/run-stepwise-ladder.sh index 849c90a..4a486c5 100755 --- a/ai-benchmark/run-stepwise-ladder.sh +++ b/ai-benchmark/run-stepwise-ladder.sh @@ -2,6 +2,7 @@ # Runs the round-2 stepwise set: every steps-ladder/<name>.json, each from a fresh project, k times. # Usage: run-stepwise-ladder.sh <tag> [k] Results: stepwise/<tag>-<i>/<name>/ (results.json, run.log, traces) # BENCH_REFERENCE=1 applies the reference files instead of asking the model (the reference pass of the steps files). +# BENCH_NEGATIVE=1 writes nothing, so each step is scored on the step before it (the negative pass: every step must fail). # BENCH_ONLY="<name> [<name> ...]" runs only those examples. # BENCH_STEPS_DIR=steps-new runs the new set (2026-10-18) instead of the ladder in steps-ladder/. set -u
