Parallel test execution: new option `parallel-tests'. 2008-07-05 Akim Demaille <[EMAIL PROTECTED]> Jim Meyering <[EMAIL PROTECTED]> Benoit Sigoure <[EMAIL PROTECTED]> Ralf Wildenhues <[EMAIL PROTECTED]>
* automake.in (handle_tests): Set new conditional PARALLEL_TESTS when reading check.am. Handle option `parallel-tests' and all its new semantics. Define macros TEST_SUFFIXES, TEST_SUITE_LOG, TEST_SUITE_HTML, TEST_LOGS, TEST_LOGS_TMP, suffix rules if applicable, and per-target rules for other tests. Add all log files to %clean_files at the `MOSTLY_CLEAN' level. * lib/Automake/Options.pm (_process_option_list): Accept `parallel-tests'. * lib/am/check.am [!PARALLEL_TESTS] (check-TESTS): Move existing testsuite driver under this new conditional. [PARALLEL_TESTS] (am__rst_title, am__rst_section, am__text_box am__sh_e_setup) (am__check_pre, am__check_post): New internal macros. ($(TEST_SUITE_LOG), check-TESTS, .log.html, check-html): New rules. * lib/am/check2.am: New file. * lib/am/Makefile.am (dist_am_DATA): Add check2.am. * tests/Makefile.am (AUTOMAKE_OPTIONS): Use `parallel-tests'. (clean-local): Renamed from distclean-local. * tests/defs.in: Drop VERBOSE handling, not needed here any more. diff --git a/automake.in b/automake.in index 99f6400..b4e0312 100755 --- a/automake.in +++ b/automake.in @@ -4738,13 +4738,95 @@ sub handle_tests { push (@check_tests, 'check-TESTS'); $output_rules .= &file_contents ('check', new Automake::Location, - COLOR => !! option 'color-tests'); + COLOR => !! option 'color-tests', + PARALLEL_TESTS => !! option 'parallel-tests'); # Tests that are known programs should have $(EXEEXT) appended. # For matching purposes, we need to adjust XFAIL_TESTS as well. append_exeext { exists $known_programs{$_[0]} } 'TESTS'; append_exeext { exists $known_programs{$_[0]} } 'XFAIL_TESTS' if (var ('XFAIL_TESTS')); + + if (option 'parallel-tests') + { + define_variable ('TEST_SUITE_LOG', 'test-suite.log', INTERNAL); + define_variable ('TEST_SUITE_HTML', '$(TEST_SUITE_LOG:.log=.html)', INTERNAL); + my $suff = '.test'; + my $at_exeext = ''; + if (exists $configure_vars{'EXEEXT'}) + { + $at_exeext = subst ('EXEEXT'); + $suff = $at_exeext . ' ' . $suff; + } + define_variable ('TEST_SUFFIXES', $suff, INTERNAL); + # FIXME: this mishandles conditions. + my @test_suffixes = (var 'TEST_SUFFIXES')->value_as_list_recursive; + if (exists $configure_vars{'EXEEXT'}) + { + unshift (@test_suffixes, $at_exeext) + unless $test_suffixes[0] eq $at_exeext; + } + unshift (@test_suffixes, ''); + + transform_variable_recursively + ('TESTS', 'TEST_LOGS', 'am__testlogs', 1, INTERNAL, + sub { + my ($subvar, $val, $cond, $full_cond) = @_; + my $obj = $val; + return $obj + if $val =~ /[EMAIL PROTECTED]@$/; + $obj =~ s/\$\(EXEEXT\)$//o; + foreach my $test_suffix (@test_suffixes) + { + next + if $test_suffix eq $at_exeext || $test_suffix eq ''; + return substr ($obj, 0, length ($obj) - length ($test_suffix)) . '.log' + if substr ($obj, - length ($test_suffix)) eq $test_suffix; + } + $obj .= '.log'; + $output_rules .= file_contents ('check2', new Automake::Location, + GENERIC => 0, + OBJ => $obj, + SOURCE => $val, + EXT => ''); + return $obj; + }); + + my $nhelper=1; + my $prev = 'TESTS'; + my $post = ''; + my $last_suffix = $test_suffixes[$#test_suffixes]; + my $cur = ''; + foreach my $test_suffix (@test_suffixes) + { + if ($test_suffix eq $last_suffix) + { + $cur = 'TEST_LOGS'; + } + else + { + $cur = 'am__test_logs' . $nhelper; + } + define_variable ($cur, + '$(' . $prev . ':' . $test_suffix . $post . '=.log)', INTERNAL); + $post = '.log'; + $prev = $cur; + $nhelper++; + $output_rules .= file_contents ('check2', new Automake::Location, + GENERIC => 1, + OBJ => '', + SOURCE => '$<', + EXT => $test_suffix) + if $test_suffix ne $at_exeext && $test_suffix ne ''; + } + + define_variable ('TEST_LOGS_TMP', '$(TEST_LOGS:.log=.log-t)', INTERNAL); + + $clean_files{'$(TEST_LOGS_TMP)'} = MOSTLY_CLEAN; + $clean_files{'$(TEST_LOGS)'} = MOSTLY_CLEAN; + $clean_files{'$(TEST_SUITE_LOG)'} = MOSTLY_CLEAN; + $clean_files{'$(TEST_SUITE_HTML)'} = MOSTLY_CLEAN; + } } } diff --git a/lib/Automake/Options.pm b/lib/Automake/Options.pm index 1705981..df08e05 100644 --- a/lib/Automake/Options.pm +++ b/lib/Automake/Options.pm @@ -266,7 +266,7 @@ sub _process_option_list (\%$@) || $_ eq 'subdir-objects' || $_ eq 'nostdinc' || $_ eq 'no-exeext' || $_ eq 'no-define' || $_ eq 'std-options' - || $_ eq 'color-tests' + || $_ eq 'color-tests' || $_ eq 'parallel-tests' || $_ eq 'cygnus' || $_ eq 'no-dependencies') { # Explicitly recognize these. diff --git a/lib/am/Makefile.am b/lib/am/Makefile.am index f6fdafa..1ab2f31 100644 --- a/lib/am/Makefile.am +++ b/lib/am/Makefile.am @@ -2,7 +2,7 @@ ## Makefile for Automake lib/am. -## Copyright (C) 1995, 1996, 1997, 1998, 1999, 2001, 2003, 2004 +## Copyright (C) 1995, 1996, 1997, 1998, 1999, 2001, 2003, 2004, 2008 ## Free Software Foundation, Inc. ## This program is free software; you can redistribute it and/or modify @@ -23,6 +23,7 @@ amdir = $(pkgvdatadir)/am dist_am_DATA = \ ansi2knr.am \ check.am \ +check2.am \ clean-hdr.am \ clean.am \ compile.am \ diff --git a/lib/am/check.am b/lib/am/check.am index ca400fe..129f2d1 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -38,6 +38,219 @@ endif !%?COLOR% .PHONY: check-TESTS +if %?PARALLEL_TESTS% + +include inst-vars.am + +## New parallel test driver. +## +## This provides special support for "unit tests", that is to +## say, tests that (once run) no longer need to be re-compiled and +## re-run at each "make check", unless their sources changed. To +## enable unit-test supports, define LAZY_TEST_SUITE. In such a +## setting, that heavily relies on correct dependencies, its users may +## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because +## it allows intertwined compilation and execution of the tests. +## Sometimes this helps catching errors earlier (you don't have to +## wait for all the tests to be compiled). +## +## Define TEST_SUITE_LOG to be the name of the global log to create. +## Define TEST_LOGS to the set of logs to include in it. It defaults +## to $(TESTS), with `.test' and [EMAIL PROTECTED]@' removed, and `'.log' +## appended. +## +## In addition to the magic "exit 77 means SKIP" feature (which was +## imported from automake), there is a magic "exit 99 means FAIL" feature +## which is useful if you need to issue a hard error no matter whether the +## test is XFAIL or not. You can disable this feature by setting the +## variable DISABLE_HARD_ERRORS to `yes'. + +# Restructured Text title and section. +am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//' +am__rst_section = sed 'p;s/./=/g;p;g' + +# Put stdin (possibly several lines separated by ". ") in a box. +am__text_box = sed 's/\. /\n/g' | sed '/^$$/d' | \ +$(AWK) '{ if (final) final = final "\n" $$0; else final = $$0; }\ +max < length($$0) { max = length($$0); } \ +END { \ + for (i = 0; i < max; ++i) line = line "="; \ + print line; print final; print line; \ +}' + +# Solaris 10 'make', and several other traditional 'make' implementations, +# pass "-e" to $(SHELL). This contradicts POSIX. Work around the problem +# by disabling -e (using the XSI extension "set +e") if it's set. +am__sh_e_setup = case $$- in *e*) set +e;; esac + +# To be inserted before the command running the test. Creates the +# directory for the log if needed. Stores in $dir the directory +# containing $f, in $tst the test, in $log the log, and passes +# TESTS_ENVIRONMENT. Save and restore TERM around use of +# TESTS_ENVIRONMENT, in case that unsets it. +am__check_pre = \ +$(am__sh_e_setup); \ +$(am__vpath_adj_setup) $(am__vpath_adj) \ +srcdir=$(srcdir); export srcdir; \ +rm -f [EMAIL PROTECTED]; \ +trap 'st=$$?; rm -f '\''$(abs_builddir)/[EMAIL PROTECTED]'\''; (exit $$st); exit $$st' \ + 1 2 13 15; \ +am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \ +test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \ +if test -f "./$$f"; then dir=./; \ +elif test -f "$$f"; then dir=; \ +else dir="$(srcdir)/"; fi; \ +tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \ +$(TESTS_ENVIRONMENT) + +# To be appended to the command running the test. Handle the stdout +# and stderr redirection, and catch the exit status. +am__check_post = \ +>[EMAIL PROTECTED] 2>&1; \ +estatus=$$?; \ +TERM=$$__SAVED_TERM; export TERM; \ +$(am__tty_colors); \ +xfailed=PASS; \ +for xfail in : $(XFAIL_TESTS); do \ + case $$f in \ + $$xfail | $(srcdir)/$$xfail) xfailed=XFAIL; break; \ + esac; \ +done; \ +case $$estatus:$$xfailed in \ + 0:XFAIL) col=$$red; res=XPASS;; \ + 0:*) col=$$grn; res=PASS ;; \ + 77:*) col=$$blu; res=SKIP ;; \ + 99:*) col=$$red; res=FAIL ;; \ + *:XFAIL) col=$$lgn; res=XFAIL;; \ + *:*) col=$$red; res=FAIL ;; \ +esac; \ +echo "$${col}$$res$${std}: $$f"; \ +echo "$$res: $$f (exit: $$estatus)" | \ + $(am__rst_section) >$@; \ +cat [EMAIL PROTECTED] >>$@; \ +rm -f [EMAIL PROTECTED]; \ +if test -z '$(DISABLE_HARD_ERRORS)' \ + && test $$estatus -eq 99; then \ + exit $$estatus; else :; \ +fi + +$(TEST_SUITE_LOG): $(TEST_LOGS) + @$(am__sh_e_setup); \ + list='$(TEST_LOGS)'; \ + results=`for f in $$list; do \ + read line < $$f && echo "$$line" || echo FAIL; \ + done`; \ + all=`echo "$$results" | wc -l | sed -e 's/^[ ]*//'`; \ + fail=`echo "$$results" | grep -c '^FAIL'`; \ + pass=`echo "$$results" | grep -c '^PASS'`; \ + skip=`echo "$$results" | grep -c '^SKIP'`; \ + xfail=`echo "$$results" | grep -c '^XFAIL'`; \ + xpass=`echo "$$results" | grep -c '^XPASS'`; \ + failures=`expr $$fail + $$xpass`; \ + all=`expr $$all - $$skip`; \ + if test "$$all" -eq 1; then tests=test; All=; \ + else tests=tests; All="All "; fi; \ + case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \ + fail=0:xpass=0:xfail=0) \ + msg="$$All$$all $$tests passed. "; \ + exit=true;; \ + fail=0:xpass=0:xfail=*) \ + msg="$$All$$all $$tests behaved as expected"; \ + if test "$$xfail" -eq 1; then xfailures=failure; \ + else xfailures=failures; fi; \ + msg="$$msg ($$xfail expected $$xfailures). "; \ + exit=true;; \ + fail=*:xpass=0:xfail=*) \ + msg="$$fail of $$all $$tests failed. "; \ + exit=false;; \ + fail=*:xpass=*:xfail=*) \ + msg="$$failures of $$all $$tests did not behave as expected"; \ + if test "$$xpass" -eq 1; then xpasses=pass; \ + else xpasses=passes; fi; \ + msg="$$msg ($$xpass unexpected $$xpasses). "; \ + exit=false;; \ + *) \ + echo >&2 "incorrect case"; exit 4;; \ + esac; \ + if test "$$skip" -ne 0; then \ + if test "$$skip" -eq 1; then \ + msg="$$msg($$skip test was not run). "; \ + else \ + msg="$$msg($$skip tests were not run). "; \ + fi; \ + fi; \ + if test "$$failures" -ne 0; then \ + { \ + echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ + $(am__rst_title); \ + echo "$$msg"; \ + echo; \ + echo ".. contents:: :depth: 2"; \ + echo; \ + for f in $$list; do \ + read line < $$f; \ + case $$line in \ + SKIP:*|PASS:*|XFAIL:*);; \ + *) echo; cat $$f;; \ + esac; \ + done; \ + } >$(TEST_SUITE_LOG).tmp; \ + mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ + msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \ + if test -n "$(PACKAGE_BUGREPORT)"; then \ + msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \ + fi; \ + fi; \ + $(am__tty_colors); \ + if $$exit; then \ + echo $(ECHO_N) "$$grn$(ECHO_C)"; \ + else \ + echo $(ECHO_N) "$$red$(ECHO_C)"; \ + fi; \ + echo "$$msg" | $(am__text_box); \ + echo $(ECHO_N) "$$std$(ECHO_C)"; \ + test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \ + $$exit + +# Run all the tests. +check-TESTS: + @if test -z '$(LAZY_TEST_SUITE)' \ + && test -n "$(TEST_SUITE_LOG)$(TEST_LOGS)"; then \ + rm -f $(TEST_SUITE_LOG) $(TEST_LOGS); \ + fi + @$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) + + +## -------------- ## +## Produce HTML. ## +## -------------- ## + +.log.html: + @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \ + for r2h in $$list; do \ + if ($$r2h --version) >/dev/null 2>&1; then \ + R2H=$$r2h; \ + fi; \ + done; \ + if test -z "$$R2H"; then \ + echo >&2 "cannot find rst2html, cannot create $@"; \ + exit 2; \ + fi; \ + $$R2H $< >[EMAIL PROTECTED] + @mv [EMAIL PROTECTED] $@ + +# Be sure to run check-TESTS first, and then to convert the result. +# Beware of concurrent executions. And expect check-TESTS to fail. +check-html: + @if $(MAKE) $(AM_MAKEFLAGS) check-TESTS; then :; else \ + rv=$$?; \ + $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML); \ + exit $$rv; \ + fi +.PHONY: check-html + +else !%?PARALLEL_TESTS% + check-TESTS: $(TESTS) @failed=0; all=0; xfail=0; xpass=0; skip=0; \ srcdir=$(srcdir); export srcdir; \ @@ -138,3 +351,5 @@ check-TESTS: $(TESTS) echo "$$dashes$$std"; \ test "$$failed" -eq 0; \ else :; fi + +endif !%?PARALLEL_TESTS% diff --git a/lib/am/check2.am b/lib/am/check2.am new file mode 100644 index 0000000..58ca9e6 --- /dev/null +++ b/lib/am/check2.am @@ -0,0 +1,20 @@ +## automake - create Makefile.in from Makefile.am +## Copyright (C) 2008 Free Software Foundation, Inc. + +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3, or (at your option) +## any later version. + +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with this program. If not, see <http://www.gnu.org/licenses/>. + +## From a test file to a log file. +?GENERIC?%EXT%.log: +?!GENERIC?%OBJ%: %SOURCE% + @p='%SOURCE%'; $(am__check_pre) "$$tst" $(am__check_post) diff --git a/tests/.gitignore b/tests/.gitignore index f1c3bff..a6827fd 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -2,3 +2,5 @@ aclocal-* automake-* defs *.dir +*.log +*.log-t diff --git a/tests/Makefile.am b/tests/Makefile.am index 32cefa5..9cc6dbe 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,4 +1,5 @@ ## Process this file with automake to create Makefile.in +AUTOMAKE_OPTIONS = parallel-tests XFAIL_TESTS = \ all.test \ @@ -670,7 +671,7 @@ EXTRA_DIST = ChangeLog-old $(TESTS) # Each test case depends on defs, aclocal, and automake. check_SCRIPTS = defs aclocal-$(APIVERSION) automake-$(APIVERSION) -distclean-local: check-clean-local +clean-local: check-clean-local check-clean-local: -chmod -R u+rwx *.dir diff --git a/tests/defs.in b/tests/defs.in index fb4e0e0..063ce58 100644 --- a/tests/defs.in +++ b/tests/defs.in @@ -40,9 +40,7 @@ test -f ./defs || { exit 1 } -# If srcdir is not set, then we are not running from `make check', be verbose. if test -z "$srcdir"; then - VERBOSE=x # compute $srcdir. srcdir=`echo "$0" | sed -e 's,/[^\\/]*$,,'` test "$srcdir" = $0 && srcdir=. @@ -56,12 +54,6 @@ test -f "$srcdir/defs.in" || { me=`echo "$0" | sed -e 's,.*[\\/],,;s/\.test$//'` -# See how redirections should work. User can set VERBOSE to see all -# output. -test -z "$VERBOSE" && { - exec > /dev/null 2>&1 -} - # Make sure we override the user shell. SHELL='@SHELL@' export SHELL @@ -402,10 +394,7 @@ AUTOMAKE_fails () AUTOMAKE_run 1 ${1+"$@"} } -# Turn on shell traces when VERBOSE is set. -if test -n "$VERBOSE"; then - set -x -else - : -fi +# Turn on shell traces. +set -x + pwd