This is an automated email from the ASF dual-hosted git repository. rombert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/sling-whiteboard.git
commit 235bb99d4239d9208b83e0cc24eb0f083795ccc9 Author: Robert Munteanu <[email protected]> AuthorDate: Wed Apr 22 17:51:04 2026 +0200 feat(skill_evals): test updating to parent pom 27 with customer scorer --- .../src/skill_evals/jcr_js_nodetypes/dataset.jsonl | 1 + .../src/skill_evals/jcr_js_nodetypes/scorer.py | 59 ++++++++++++++++++---- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl b/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl index dc55984b..557cd0f3 100644 --- a/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl +++ b/skill-evals/src/skill_evals/jcr_js_nodetypes/dataset.jsonl @@ -1,2 +1,3 @@ {"id":"upgrade-parent-pom-{expected_parent_version}","input":"Update to parent pom version {expected_parent_version}","git_revision":"52cea70b190c8389495a4fa494161eac13416891","expected_parent_version":"23"} {"id":"upgrade-parent-pom-{expected_parent_version}","input":"Update to parent pom version {expected_parent_version}","git_revision":"84554fffb6c46bf48bb1856a6e60852127f57bcd","expected_parent_version":"24"} +{"id":"upgrade-parent-pom-{expected_parent_version}","input":"Update to parent pom version {expected_parent_version}","git_revision":"55f0f73e9442e13cad0ab9b1ca42e460abeab162","expected_parent_version":"27"} diff --git a/skill-evals/src/skill_evals/jcr_js_nodetypes/scorer.py b/skill-evals/src/skill_evals/jcr_js_nodetypes/scorer.py index 1989e1bc..20d6a9c1 100644 --- a/skill-evals/src/skill_evals/jcr_js_nodetypes/scorer.py +++ b/skill-evals/src/skill_evals/jcr_js_nodetypes/scorer.py @@ -8,6 +8,35 @@ from inspect_ai.util import sandbox REPO_DIR = "/workspace/sling-org-apache-sling-jcr-js-nodetypes" POM_PATH = f"{REPO_DIR}/pom.xml" PARENT_VERSION_RE = re.compile(r"<parent>.*?<version>([^<]+)</version>", re.DOTALL) +REQUIRED_PARENT_POM_27_DEPENDENCIES = ( + "org.osgi:osgi.cmpn:jar:6.0.0:provided", + "org.osgi:osgi.core:jar:6.0.0:provided", + "javax.servlet:javax.servlet-api:jar:3.1.0:provided", +) + + +async def check_parent_pom_27_dependencies() -> tuple[int, str]: + dependency_result = await sandbox().exec( + ["mvn", "dependency:list"], + cwd=REPO_DIR, + timeout=600, + timeout_retry=False, + ) + dependency_output = "\n".join( + part for part in [dependency_result.stdout, dependency_result.stderr] if part + ) + missing_dependencies = [ + dependency + for dependency in REQUIRED_PARENT_POM_27_DEPENDENCIES + if dependency not in dependency_output + ] + dependency_check_passed = 1 if dependency_result.success and not missing_dependencies else 0 + parent_pom_27_dependency_check = ( + "passed" + if dependency_check_passed + else f"missing: {', '.join(missing_dependencies)}" + ) + return dependency_check_passed, parent_pom_27_dependency_check @scorer(metrics=[accuracy(), stderr()]) @@ -28,9 +57,17 @@ def parent_pom_update() -> Scorer: timeout_retry=False, ) + dependency_check_passed = 1 + parent_pom_27_dependency_check = None + if expected_version == "27": + ( + dependency_check_passed, + parent_pom_27_dependency_check, + ) = await check_parent_pom_27_dependencies() + parent_version_correct = 1 if actual_version == expected_version else 0 build_passed = 1 if build_result.success else 0 - overall = 1 if parent_version_correct and build_passed else 0 + overall = 1 if parent_version_correct and build_passed and dependency_check_passed else 0 stderr_tail = build_result.stderr.strip() if stderr_tail: @@ -45,18 +82,22 @@ def parent_pom_update() -> Scorer: if stderr_tail: explanation += f"mvn stderr (tail):\n{stderr_tail}\n" + metadata = { + "expected_parent_version": expected_version, + "actual_parent_version": actual_version, + "mvn_returncode": build_result.returncode, + "parent_version_correct": parent_version_correct, + "build_passed": build_passed, + "overall": overall, + } + if parent_pom_27_dependency_check is not None: + metadata["parent_pom_27_dependency_check"] = parent_pom_27_dependency_check + return Score( value=overall, answer=f"parent_version={actual_version}, build_success={build_result.success}", explanation=explanation, - metadata={ - "expected_parent_version": expected_version, - "actual_parent_version": actual_version, - "mvn_returncode": build_result.returncode, - "parent_version_correct": parent_version_correct, - "build_passed": build_passed, - "overall": overall, - }, + metadata=metadata, ) return score
