While testing my previous patch for const/volatile vs. -Werror, I encountered two autotest anomalies. I first enhanced the testsuite to detect both problems, as well as my earlier patch this week for when removing the test group directory fails.
The first bug was that when using the -d option to testsuite (which is done by all the testsuite.dir/###/run scripts), the run script was only regenerated if the test failed (it was missing after a success or skip). I also managed to come up with a patch for this. The second bug is that environment variables passed as options to ./testsuite are copied directly into the run scripts, but without regards for shell metacharacters. That means that an environment variable that contains spaces, such as "./run CFLAGS='-std=c89 -Werror'", will regenerate a broken run script that tries to invoke 'testsuite CFLAGS=-std=c89 -Werror'. However, fixing this quickly and safely is beyond my abilities, with a pending alpha release, so the test is just xfailed for now. 2006-04-07 Eric Blake <[EMAIL PROTECTED]> * tests/autotest.at (AT_CHECK_AT_START): New macro, split out from... (AT_CHECK_AT): ...here, for use in... (Debugging a successful test, Debugging a failed test): ...these new tests. The first of these is fixed by... * lib/autotest/general.m4 (_AT_CREATE_DEBUGGING_SCRIPT): New macro, split out from... (AT_INIT): ...here, so that using -d also generates a run script.
Index: lib/autotest/general.m4 =================================================================== RCS file: /sources/autoconf/autoconf/lib/autotest/general.m4,v retrieving revision 1.206 diff -u -p -r1.206 general.m4 --- lib/autotest/general.m4 7 Apr 2006 18:31:09 -0000 1.206 +++ lib/autotest/general.m4 8 Apr 2006 00:57:29 -0000 @@ -162,6 +162,22 @@ m4_define([_AT_NORMALIZE_TEST_GROUP_NUMB done ]) +# _AT_CREATE_DEBUGGING_SCRIPT +# --------------------------- +# Create the debugging script $at_group_dir/run which will reproduce the +# current test group. +m4_define([_AT_CREATE_DEBUGGING_SCRIPT], +[ { + echo "#! /bin/sh" + echo 'test "${ZSH_VERSION+set}" = set && alias -g '\''${1+"[EMAIL PROTECTED]"}'\''='\''"[EMAIL PROTECTED]"'\''' + echo "cd '$at_dir'" + echo 'exec ${CONFIG_SHELL-'"$SHELL"'}' "$[0]" \ + '-v -d' "$at_debug_args" "$at_group" '${1+"[EMAIL PROTECTED]"}' + echo 'exit 1' + } >$at_group_dir/run + chmod +x $at_group_dir/run +])# _AT_CREATE_DEBUGGING_SCRIPT + # AT_INIT([TESTSUITE-NAME]) # ------------------------- @@ -793,11 +809,12 @@ _ATEOF echo "$at_log_msg" >&AS_MESSAGE_LOG_FD # Cleanup the group directory, unless the user wants the files. - $at_debug_p || - if test -d $at_group_dir; then - find $at_group_dir -type d ! -perm -700 -exec chmod u+rwx \{\} \; - rm -fr $at_group_dir - fi + if $at_debug_p ; then + _AT_CREATE_DEBUGGING_SCRIPT + elif test -d $at_group_dir; then + find $at_group_dir -type d ! -perm -700 -exec chmod u+rwx \{\} \; + rm -fr $at_group_dir + fi ;; *) # Upon failure, include the log into the testsuite's global @@ -807,15 +824,7 @@ _ATEOF # Upon failure, keep the group directory for autopsy, and # create the debugging script. - { - echo "#! /bin/sh" - echo 'test "${ZSH_VERSION+set}" = set && alias -g '\''${1+"[EMAIL PROTECTED]"}'\''='\''"[EMAIL PROTECTED]"'\''' - echo "cd '$at_dir'" - echo 'exec ${CONFIG_SHELL-'"$SHELL"'}' "$[0]" \ - '-v -d' "$at_debug_args" "$at_group" '${1+"[EMAIL PROTECTED]"}' - echo 'exit 1' - } >$at_group_dir/run - chmod +x $at_group_dir/run + _AT_CREATE_DEBUGGING_SCRIPT $at_errexit && break ;; esac Index: tests/autotest.at =================================================================== RCS file: /sources/autoconf/autoconf/tests/autotest.at,v retrieving revision 1.11 diff -u -p -r1.11 autotest.at --- tests/autotest.at 5 Apr 2006 16:04:22 -0000 1.11 +++ tests/autotest.at 8 Apr 2006 00:57:29 -0000 @@ -19,14 +19,15 @@ AT_BANNER([Autotest.]) # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. -# AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0], -# [STDOUT := ignore], STDERR) -# --------------------------------------------------------------- +# AT_CHECK_AT_START(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0], +# [STDOUT := ignore], STDERR) +# --------------------------------------------------------------------- # Create a new test named TITLE that runs a minimal Autotest test suite, # SUITE-CODE. Call AT_XFAIL_IF with XFAIL-CONDITION. STATUS and STDERR pass # directly to the AT_CHECK that call the minimal test suite. STDOUT is not -# used, but it is reserved for future use. -m4_define([AT_CHECK_AT], +# used, but it is reserved for future use. The test group remains open +# for further testing before AT_CLEANUP. +m4_define([AT_CHECK_AT_START], [ AT_SETUP([$1]) AT_KEYWORDS([autotest]) @@ -49,6 +50,17 @@ AT_DATA([mysuite.at], [$2]) AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at]) AT_CHECK([$CONFIG_SHELL ./micro-suite], m4_default([$4], 0), [ignore], [$6]) AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], [$6]) +])# AT_CHECK_AT_START + +# AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0], +# [STDOUT := ignore], STDERR) +# --------------------------------------------------------------- +# Create a new test named TITLE that runs a minimal Autotest test suite, +# SUITE-CODE. Call AT_XFAIL_IF with XFAIL-CONDITION. STATUS and STDERR pass +# directly to the AT_CHECK that call the minimal test suite. STDOUT is not +# used, but it is reserved for future use. +m4_define([AT_CHECK_AT], +[AT_CHECK_AT_START($@) AT_CLEANUP ]) @@ -251,6 +263,41 @@ AT_CHECK_AT_TITLE_CHAR([Double-quote], [ AT_CHECK_AT_TITLE_CHAR([Backslash], [\]) +## ----------------- ## +## Debugging a test. ## +## ----------------- ## + +# At one point, using the -d switch failed to rebuild the run script. +# Also, running from in the group dir locks the directory in some platforms. +AT_CHECK_AT_START([Debugging a successful test], +[[ +AT_INIT([artificial test suite]) +AT_SETUP([Environment check]) +AT_CHECK([test "$MY_VAR" = "two spaces"]) +AT_CLEANUP +]], [], [1], [], [ignore]) +AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='two spaces')], + [0], [ignore], [ignore]) +AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='one space')], + [1], [ignore], [ignore]) +AT_CLEANUP + +# The run script is currently invalid when shell metacharacters are passed +# in via an environment option. +AT_CHECK_AT_START([Debugging a failed test], +[[ +AT_INIT([artificial test suite]) +AT_SETUP([Environment check]) +AT_CHECK([test "$MY_VAR" = "one space"]) +AT_CLEANUP +]], [:], [1], [], [ignore]) +AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='two spaces')], + [1], [ignore], [ignore]) +AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='one space')], + [0], [ignore], [ignore]) +AT_CLEANUP + + ## --------- ## ## Keywords. ## ## --------- ##