Module Name: src
Committed By: rillig
Date: Sat Jul 4 22:01:10 UTC 2020
Modified Files:
src/usr.bin/make/unit-tests: Makefile varshell.exp
Log Message:
make(1): add post-processing for tests
By defining SED_CMDS.${test}, a test can do additional post-processing
on its output, for example to normalize paths or error messages.
The Makefile for the tests now follows the common style to have all
relevant definitions at the top of the file, and all the implementation
details at the bottom.
By using the += instead of backslashed lists, it becomes easier to
comment out a single test, either temporarily or permanently (such as
the POSIX tests).
The :C modifier has been replaced with the simpler :S, some :S modifiers
have been replaced with simpler := modifiers.
The postprocessing commands have been extracted from the actual command.
This makes editing and commenting them easier.
To generate a diff of this commit:
cvs rdiff -u -r1.60 -r1.61 src/usr.bin/make/unit-tests/Makefile
cvs rdiff -u -r1.2 -r1.3 src/usr.bin/make/unit-tests/varshell.exp
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/usr.bin/make/unit-tests/Makefile
diff -u src/usr.bin/make/unit-tests/Makefile:1.60 src/usr.bin/make/unit-tests/Makefile:1.61
--- src/usr.bin/make/unit-tests/Makefile:1.60 Sat Jul 4 21:04:25 2020
+++ src/usr.bin/make/unit-tests/Makefile Sat Jul 4 22:01:10 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.60 2020/07/04 21:04:25 rillig Exp $
+# $NetBSD: Makefile,v 1.61 2020/07/04 22:01:10 rillig Exp $
#
# Unit tests for make(1)
#
@@ -11,83 +11,90 @@
# accept:
# move generated output to expected results
#
+# Settable variables
+#
+# TEST_MAKE
+# The make program to be tested.
+#
#
# Adding a test case
#
# Each feature should get its own set of tests in its own suitably
# named makefile (*.mk), with its own set of expected results (*.exp),
-# and it should be added to the TESTNAMES list.
+# and it should be added to the TESTS list.
#
# Any added files must also be added to src/distrib/sets/lists/tests/mi.
-# Makefiles that are not added to TESTNAMES must be ignored in
+# Makefiles that are not added to TESTS must be ignored in
# src/tests/usr.bin/make/t_make.sh (example: include-sub).
#
+# Each test is in a sub-makefile.
+# Keep the list sorted.
+TESTS+= comment
+TESTS+= cond-late
+TESTS+= cond-short
+TESTS+= cond1
+TESTS+= cond2
+TESTS+= dollar
+TESTS+= doterror
+TESTS+= dotwait
+TESTS+= error
+TESTS+= # escape # broken by referting POSIX changes
+TESTS+= export
+TESTS+= export-all
+TESTS+= export-env
+TESTS+= forloop
+TESTS+= forsubst
+TESTS+= hash
+TESTS+= # impsrc # broken by referting POSIX changes
+TESTS+= include-main
+TESTS+= misc
+TESTS+= moderrs
+TESTS+= modmatch
+TESTS+= modmisc
+TESTS+= modorder
+TESTS+= modts
+TESTS+= modword
+TESTS+= order
+TESTS+= # phony-end # broken by referting POSIX changes
+TESTS+= posix
+TESTS+= # posix1 # broken by referting POSIX changes
+TESTS+= qequals
+TESTS+= # suffixes # broken by referting POSIX changes
+TESTS+= sunshcmd
+TESTS+= sysv
+TESTS+= ternary
+TESTS+= unexport
+TESTS+= unexport-env
+TESTS+= varcmd
+TESTS+= varmisc
+TESTS+= varmod-edge
+TESTS+= varquote
+TESTS+= varshell
+
+# Override make flags for certain tests; default is -k.
+FLAGS.doterror= # none
+FLAGS.order= -j1
+
+# Some tests need extra post-processing.
+SED_CMDS.varshell+= -e 's,^[a-z]*sh: ,,'
+SED_CMDS.varshell+= -e '/command/s,No such.*,not found,'
+
+# End of the configuration section.
+
.MAIN: all
-UNIT_TESTS:= ${.PARSEDIR}
+UNIT_TESTS:= ${.PARSEDIR}
.PATH: ${UNIT_TESTS}
-# Each test is in a sub-makefile.
-# Keep the list sorted.
-TESTNAMES= \
- comment \
- cond-late \
- cond-short \
- cond1 \
- cond2 \
- dollar \
- doterror \
- dotwait \
- error \
- export \
- export-all \
- export-env \
- forloop \
- forsubst \
- hash \
- include-main \
- misc \
- moderrs \
- modmatch \
- modmisc \
- modorder \
- modts \
- modword \
- order \
- posix \
- qequals \
- sunshcmd \
- sysv \
- ternary \
- unexport \
- unexport-env \
- varcmd \
- varmisc \
- varmod-edge \
- varquote \
- varshell
-
-# these tests were broken by referting POSIX chanegs
-STRICT_POSIX_TESTS = \
- escape \
- impsrc \
- phony-end \
- posix1 \
- suffixes
-
-# Override make flags for certain tests
-flags.doterror=
-flags.order=-j1
-
-OUTFILES= ${TESTNAMES:S/$/.out/}
+OUTFILES= ${TESTS:=.out}
all: ${OUTFILES}
-CLEANFILES += *.rawout *.out *.status *.tmp *.core *.tmp
-CLEANFILES += obj*.[och] lib*.a # posix1.mk
-CLEANFILES += issue* .[ab]* # suffixes.mk
-CLEANRECURSIVE += dir dummy # posix1.mk
+CLEANFILES+= *.rawout *.out *.status *.tmp *.core *.tmp
+CLEANFILES+= obj*.[och] lib*.a # posix1.mk
+CLEANFILES+= issue* .[ab]* # suffixes.mk
+CLEANRECURSIVE+= dir dummy # posix1.mk
clean:
rm -f ${CLEANFILES}
@@ -95,34 +102,36 @@ clean:
rm -rf ${CLEANRECURSIVE}
.endif
-TEST_MAKE?= ${.MAKE}
-TOOL_SED?= sed
+TEST_MAKE?= ${.MAKE}
+TOOL_SED?= sed
# ensure consistent results from sort(1)
-LC_ALL= C
-LANG= C
+LC_ALL= C
+LANG= C
.export LANG LC_ALL
# the tests are actually done with sub-makes.
.SUFFIXES: .mk .rawout .out
.mk.rawout:
- @echo ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC}
+ @echo ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC}
-@cd ${.OBJDIR} && \
- { ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC} \
+ { ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC} \
2>&1 ; echo $$? >${.TARGET:R}.status ; } > ${.TARGET}.tmp
@mv ${.TARGET}.tmp ${.TARGET}
+_SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
+_SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,'
+_SED_CMDS+= -e '/stopped/s, /.*, unit-tests,'
+_SED_CMDS+= -e 's,${.CURDIR:S,.,\\.,g}/,,g'
+_SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
+
# We always pretend .MAKE was called 'make'
# and strip ${.CURDIR}/ from the output
# and replace anything after 'stopped in' with unit-tests
# so the results can be compared.
.rawout.out:
@echo postprocess ${.TARGET}
- @${TOOL_SED} -e 's,^${TEST_MAKE:T:C/\./\\\./g}[][0-9]*:,make:,' \
- -e 's,${TEST_MAKE:C/\./\\\./g},make,' \
- -e '/stopped/s, /.*, unit-tests,' \
- -e 's,${.CURDIR:C/\./\\\./g}/,,g' \
- -e 's,${UNIT_TESTS:C/\./\\\./g}/,,g' \
+ @${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
< ${.IMPSRC} > ${.TARGET}.tmp
@echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp
@mv ${.TARGET}.tmp ${.TARGET}
@@ -130,7 +139,7 @@ LANG= C
# Compare all output files
test: ${OUTFILES} .PHONY
@failed= ; \
- for test in ${TESTNAMES}; do \
+ for test in ${TESTS}; do \
diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
|| failed="$${failed}$${failed:+ }$${test}" ; \
done ; \
@@ -141,14 +150,14 @@ test: ${OUTFILES} .PHONY
fi
accept:
- @for test in ${TESTNAMES}; do \
+ @for test in ${TESTS}; do \
cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
|| { echo "Replacing $${test}.exp" ; \
cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
done
.if exists(${TEST_MAKE})
-${TESTNAMES:S/$/.rawout/}: ${TEST_MAKE}
+${TESTS:=.rawout}: ${TEST_MAKE}
.endif
.-include <bsd.obj.mk>
Index: src/usr.bin/make/unit-tests/varshell.exp
diff -u src/usr.bin/make/unit-tests/varshell.exp:1.2 src/usr.bin/make/unit-tests/varshell.exp:1.3
--- src/usr.bin/make/unit-tests/varshell.exp:1.2 Fri Apr 10 20:41:59 2015
+++ src/usr.bin/make/unit-tests/varshell.exp Sat Jul 4 22:01:10 2020
@@ -1,4 +1,4 @@
-sh: /bin/no/such/command: not found
+/bin/no/such/command: not found
make: "varshell.mk" line 5: warning: "/bin/no/such/command" returned non-zero status
make: "varshell.mk" line 6: warning: "kill -14 $$" exited on a signal
make: "varshell.mk" line 7: warning: "false" returned non-zero status
