This is an automated email from the ASF dual-hosted git repository.
rombert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sling-whiteboard.git
The following commit(s) were added to refs/heads/master by this push:
new d2ebd640 feat(skill-evals): add experimental tool to compare multiple
execution runs
d2ebd640 is described below
commit d2ebd6406306bcd658812c9c545d3af8bb9faaff
Author: Robert Munteanu <[email protected]>
AuthorDate: Thu Apr 23 15:32:56 2026 +0200
feat(skill-evals): add experimental tool to compare multiple execution runs
---
skill-evals/README.md | 49 ++++++++++++++
skill-evals/pyproject.toml | 3 +
skill-evals/src/skill_evals/compare_eval_runs.py | 85 ++++++++++++++++++++++++
3 files changed, 137 insertions(+)
diff --git a/skill-evals/README.md b/skill-evals/README.md
index 925a7e02..94a0bd2d 100644
--- a/skill-evals/README.md
+++ b/skill-evals/README.md
@@ -76,6 +76,55 @@ Logs can be inspected afterwards with
uv run inspect view
```
+## Comparing Eval Runs
+
+Use the reusable comparison utility to compare two Inspect log files by
configuration, score,
+execution time, and token usage.
+
+```bash
+uv run skill-evals-compare LOG_A LOG_B
+```
+
+Supported output formats:
+
+```bash
+uv run skill-evals-compare LOG_A LOG_B --format text
+uv run skill-evals-compare LOG_A LOG_B --format json
+uv run skill-evals-compare LOG_A LOG_B --format markdown
+```
+
+Compare runs that are expected to differ only by `skill_enabled`:
+
+```bash
+uv run skill-evals-compare \
+ "logs/2026-04-23T09-16-03-00-00_jcr-js-nodetypes
[skills]_LLH4FxC9xVenX3nFv5fubf.eval" \
+ "logs/2026-04-23T09-35-29-00-00_jcr-js-nodetypes [no
skills]_L3rf4MQBPZ5LKYd574wucQ.eval" \
+ --expect-diff skill_enabled
+```
+
+Compare runs that use the same parameters but different models:
+
+```bash
+uv run skill-evals-compare \
+ "logs/2026-04-22T15-52-45-00-00_jcr-js-nodetypes
[skills]_SS3oqPNNabLYozQvEQs6U2.eval" \
+ "logs/2026-04-23T09-16-03-00-00_jcr-js-nodetypes
[skills]_LLH4FxC9xVenX3nFv5fubf.eval" \
+ --expect-diff model
+```
+
+Useful options:
+
+```bash
+uv run skill-evals-compare LOG_A LOG_B --samples
+uv run skill-evals-compare LOG_A LOG_B --expect-diff model,skill_enabled
+uv run skill-evals-compare LOG_A LOG_B --headline-score parent_pom_update
+uv run skill-evals-compare LOG_A LOG_B --fail-on-unexpected-diff
+```
+
+`--samples` adds dataset-entry comparison aggregated across all epochs for
each sample id.
+
+If a run contains multiple scores, use `--headline-score` to choose which
scorer or score record
+is used for the headline score and stderr rows.
+
## Task Notes
Current status:
diff --git a/skill-evals/pyproject.toml b/skill-evals/pyproject.toml
index b1852382..c94b4df8 100644
--- a/skill-evals/pyproject.toml
+++ b/skill-evals/pyproject.toml
@@ -13,6 +13,9 @@ dependencies = [
[project.entry-points.inspect_ai]
skill_evals = "skill_evals._registry"
+[project.scripts]
+skill-evals-compare = "skill_evals.compare_eval_runs:main"
+
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
diff --git a/skill-evals/src/skill_evals/compare_eval_runs.py
b/skill-evals/src/skill_evals/compare_eval_runs.py
new file mode 100644
index 00000000..c056d94a
--- /dev/null
+++ b/skill-evals/src/skill_evals/compare_eval_runs.py
@@ -0,0 +1,85 @@
+import argparse
+import sys
+from pathlib import Path
+
+from skill_evals.compare_eval_render import render_json, render_markdown,
render_text
+from skill_evals.compare_eval_report import build_report
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Compare two Inspect eval runs by score, time, usage, and
configuration."
+ )
+ parser.add_argument("left_log", help="Path or file:// URI for the left
eval log")
+ parser.add_argument("right_log", help="Path or file:// URI for the right
eval log")
+ parser.add_argument(
+ "--expect-diff",
+ default="",
+ help=(
+ "Comma-separated logical coordinates allowed to differ, e.g. "
+ "skill_enabled,model"
+ ),
+ )
+ parser.add_argument(
+ "--format",
+ choices=("text", "json", "markdown"),
+ default="text",
+ help="Output format",
+ )
+ parser.add_argument(
+ "--samples",
+ action="store_true",
+ help="Include per-sample and per-epoch comparison details",
+ )
+ parser.add_argument(
+ "--headline-score",
+ default="",
+ help=(
+ "Score key to use for headline score reporting when runs contain
multiple "
+ "scores"
+ ),
+ )
+ parser.add_argument(
+ "--fail-on-unexpected-diff",
+ action="store_true",
+ help="Exit with status 1 when differences outside --expect-diff are
found",
+ )
+ return parser.parse_args()
+
+
+def _normalize_log_path(value: str) -> str:
+ if value.startswith("file://"):
+ return value
+ return str(Path(value).expanduser().resolve())
+
+
+def main() -> int:
+ args = _parse_args()
+ left_path = _normalize_log_path(args.left_log)
+ right_path = _normalize_log_path(args.right_log)
+ expected_diff = {item.strip() for item in args.expect_diff.split(",") if
item.strip()}
+ headline_score = args.headline_score.strip() or None
+ report = build_report(
+ left_path,
+ right_path,
+ expected_diff,
+ args.samples,
+ headline_score=headline_score,
+ )
+
+ if args.format == "json":
+ output = render_json(report)
+ elif args.format == "markdown":
+ output = render_markdown(report)
+ else:
+ output = render_text(report)
+
+ print(output)
+
+ if args.fail_on_unexpected_diff and report["unexpected_differences"]:
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
