This is an automated email from the ASF dual-hosted git repository. jungm pushed a commit to branch ee11 in repository https://gitbox.apache.org/repos/asf/tomee-tck.git
commit 3d341ca6dc867b6ea6a3b11191e96d425336ad3b Author: Markus Jung <[email protected]> AuthorDate: Sun Jul 19 01:44:06 2026 +0200 Fail invoker-driven TCK runners on hidden test and tooling failures --- runner-standalone/authentication/pom.xml | 39 ++++++++++++ runner-standalone/faces/pom.xml | 33 ++++++++++ runner-standalone/security/pom.xml | 53 ++++++++++++++++ runner-standalone/verify-invoker-result.sh | 97 ++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+) diff --git a/runner-standalone/authentication/pom.xml b/runner-standalone/authentication/pom.xml index cfde2ce..e851315 100644 --- a/runner-standalone/authentication/pom.xml +++ b/runner-standalone/authentication/pom.xml @@ -206,6 +206,45 @@ </execution> </executions> </plugin> + + <!-- Step 4: the invoker runs with ignoreFailures=true to collect the + whole baseline, so Maven exits 0 regardless of the result. With the + reviewed exclusions applied the run is expected green; aggregate the + inner surefire/failsafe reports and fail the build on any + failure/error or on a module that built but never ran its tests. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>verify-tck-result</id> + <phase>verify</phase> + <goals><goal>exec</goal></goals> + <configuration> + <skip>${tck.standalone.skip}</skip> + <executable>sh</executable> + <arguments> + <argument>${project.basedir}/../verify-invoker-result.sh</argument> + <argument>${authentication.tck.dir}</argument> + <argument>${project.build.directory}/invoker-reports</argument> + <argument>basic-authentication</argument> + <argument>custom-principal</argument> + <argument>programmatic-authentication</argument> + <argument>lifecycle</argument> + <argument>wrapping</argument> + <argument>register-session</argument> + <argument>async-authentication</argument> + <argument>status-codes</argument> + <argument>dispatching</argument> + <argument>dispatching-jsf-cdi</argument> + <argument>invoke-ejb-cdi</argument> + <argument>spi</argument> + <argument>authentication-signaturetest</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/runner-standalone/faces/pom.xml b/runner-standalone/faces/pom.xml index 694d975..e60b68b 100644 --- a/runner-standalone/faces/pom.xml +++ b/runner-standalone/faces/pom.xml @@ -169,6 +169,39 @@ </execution> </executions> </plugin> + + <!-- Step 4: the invoker runs with ignoreFailures=true (and the modules + with maven.test.failure.ignore=true) to collect the whole baseline, + so Maven exits 0 regardless of the result. With the reviewed + exclusions applied the run is expected green; aggregate the inner + surefire/failsafe reports (each include is a multi-module reactor + whose reports live two levels deeper) and fail the build on any + failure/error or on a module that built but never ran its tests. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>verify-tck-result</id> + <phase>verify</phase> + <goals><goal>exec</goal></goals> + <configuration> + <skip>${tck.standalone.skip}</skip> + <executable>sh</executable> + <arguments> + <argument>${project.basedir}/../verify-invoker-result.sh</argument> + <argument>${faces.tck.dir}</argument> + <argument>${project.build.directory}/invoker-reports</argument> + <argument>faces22</argument> + <argument>faces23</argument> + <argument>faces40</argument> + <argument>faces41</argument> + <argument>faces-signaturetest</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/runner-standalone/security/pom.xml b/runner-standalone/security/pom.xml index d6ed696..64d4bf0 100644 --- a/runner-standalone/security/pom.xml +++ b/runner-standalone/security/pom.xml @@ -217,6 +217,59 @@ </execution> </executions> </plugin> + + <!-- Step 4: the invoker runs with ignoreFailures=true to collect the + whole baseline, so Maven exits 0 regardless of the result. With the + reviewed exclusions applied the run is expected green; aggregate the + inner surefire/failsafe reports and fail the build on any + failure/error or on a module that built but never ran its tests. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>verify-tck-result</id> + <phase>verify</phase> + <goals><goal>exec</goal></goals> + <configuration> + <skip>${tck.standalone.skip}</skip> + <executable>sh</executable> + <arguments> + <argument>${project.basedir}/../verify-invoker-result.sh</argument> + <argument>${tck.dir}</argument> + <argument>${project.build.directory}/invoker-reports</argument> + <argument>app-securitycontext</argument> + <argument>app-securitycontext-auth</argument> + <argument>app-securitycontext-customprincipal</argument> + <argument>app-mem</argument> + <argument>app-db</argument> + <argument>app-ldap</argument> + <argument>app-ldap2</argument> + <argument>app-ldap3</argument> + <argument>app-custom</argument> + <argument>app-multiple-store</argument> + <argument>app-multiple-store-backup</argument> + <argument>app-mem-basic</argument> + <argument>app-mem-basic-decorate</argument> + <argument>app-mem-form</argument> + <argument>app-mem-customform</argument> + <argument>app-openid</argument> + <argument>app-openid2</argument> + <argument>app-openid3</argument> + <argument>app-custom-session</argument> + <argument>app-custom-rememberme</argument> + <argument>app-custom-identity-store-handler</argument> + <argument>app-custom-authentication-mechanism-handler</argument> + <argument>app-custom-authentication-mechanism-handler2</argument> + <argument>app-jaxrs</argument> + <argument>app-mem-policy</argument> + <argument>app-mem-policy2</argument> + <argument>security-signaturetest</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/runner-standalone/verify-invoker-result.sh b/runner-standalone/verify-invoker-result.sh new file mode 100755 index 0000000..eb7169e --- /dev/null +++ b/runner-standalone/verify-invoker-result.sh @@ -0,0 +1,97 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. + +# Fails the build when a Maven-invoker-driven TCK reactor produced test +# failures, errors, or a silently green module. +# +# verify-invoker-result.sh <reactor-root> <invoker-reports> <module>... +# +# The invoker runs with ignoreFailures=true (and the modules with +# maven.test.failure.ignore=true) so the whole compatibility baseline is +# collected in one pass; Maven therefore exits 0 regardless of the result and +# a module that dies before writing any surefire/failsafe report is silently +# green. This check is the gate that turns a red baseline back into a red +# build: +# +# - test failures/errors: summed from every surefire/failsafe TEST-*.xml and +# failsafe-summary.xml under the reactor (the raw reports the ignore flags +# do not touch), including the deployment-error case failsafe records as +# <errors> in the summary without a "Tests run" build-log line; +# - tooling failures: an expected module whose invoker BUILD result is not +# "success", or whose build log never reached the surefire/failsafe phase +# (no "Tests run" and no "No tests to run"), never wrote a report and would +# otherwise pass unnoticed. +# +# A module whose tests are all excluded legitimately reports "No tests to run" +# (or a passing surefire summary) and is accepted. + +set -eu + +reactor_root=$1 +invoker_reports=$2 +shift 2 + +status=0 + +fail() { + echo "verify-invoker-result: $1" >&2 + status=1 +} + +# --- Test failures and errors across every raw report under the reactor. --- +# TEST-*.xml carries failures="" errors="" on the <testsuite> element; +# failsafe-summary.xml carries <failures>/<errors> (and records deployment +# errors the build log does not surface as a "Tests run" line). +report_failures=0 +report_errors=0 +for report in $(find "$reactor_root" -name 'TEST-*.xml' -path '*-reports/*' 2>/dev/null); do + f=$(sed -n 's/.*<testsuite[^>]* failures="\([0-9]*\)".*/\1/p' "$report" | head -1) + e=$(sed -n 's/.*<testsuite[^>]* errors="\([0-9]*\)".*/\1/p' "$report" | head -1) + report_failures=$((report_failures + ${f:-0})) + report_errors=$((report_errors + ${e:-0})) +done +for summary in $(find "$reactor_root" -name 'failsafe-summary.xml' 2>/dev/null); do + f=$(sed -n 's/.*<failures>\([0-9]*\)<.*/\1/p' "$summary" | head -1) + e=$(sed -n 's/.*<errors>\([0-9]*\)<.*/\1/p' "$summary" | head -1) + report_failures=$((report_failures + ${f:-0})) + report_errors=$((report_errors + ${e:-0})) +done +if [ "$report_failures" -ne 0 ] || [ "$report_errors" -ne 0 ]; then + fail "$report_failures test failure(s) and $report_errors error(s) in $reactor_root" +fi + +# --- Every expected module must have run its tests. --- +for module in "$@"; do + build_result=$(sed -n 's/.*result="\([^"]*\)".*/\1/p' "$invoker_reports/BUILD-$module.xml" 2>/dev/null | head -1) + if [ "$build_result" != "success" ]; then + fail "invoker module '$module' did not build (result='${build_result:-missing}')" + continue + fi + + build_log=$(sed -n 's/.*buildlog="\([^"]*\)".*/\1/p' "$invoker_reports/BUILD-$module.xml" 2>/dev/null | head -1) + if [ -z "$build_log" ] || [ ! -f "$build_log" ]; then + fail "invoker module '$module' has no build log to verify" + continue + fi + if ! grep -q 'BUILD SUCCESS' "$build_log"; then + fail "invoker module '$module' build log does not report BUILD SUCCESS" + continue + fi + # A module that never reached surefire/failsafe wrote no report and would be + # silently green; require either executed tests or an explicit empty run. + if ! grep -qE '\[INFO\] Tests run:|No tests to run|No tests were executed' "$build_log"; then + fail "invoker module '$module' built but never ran its tests (no surefire/failsafe output)" + continue + fi +done + +if [ "$status" -ne 0 ]; then + echo "verify-invoker-result: FAILED for $reactor_root" >&2 + exit 1 +fi + +echo "verify-invoker-result: OK ($reactor_root, $# module(s), 0 failures, 0 errors)"
