* t/testsuite-recheck-speed.sh: New test. * t/testsuite-summary-speed.sh: Likewise. * t/list-of-tests.mk: Update.
The new tests are not meant to PASS/FAIL, but rather to help us to obtain quantitative measurements of the performance improvements offered by soon-to-appear optimization patches. Timing before the optimizations implemented by following patches: + Slow Debian i686 system, 1 core x 1.5 GHz, 768 MB of RAM, GNU make 3.81: - testsuite-recheck-speed.sh: 1 run, ~ 6 minutes - testsuite-summary-speed.sh: 1 run, ~ 4 minutes + Fast Solaris 10 i686 system, 4 cores x 3 GHz, 20 GB of RAM, Solaris CCS make: - testsuite-recheck-speed.sh: 4 runs, ~ 3 minutes - testsuite-summary-speed.sh: 2 runs, ~ 6 minutes + Very fast Fedora ppc64 system, 64 cores x 3.5 GHz, 64 GB of RAM, GNU make 3.82: - testsuite-recheck-speed.sh: 5 runs, ~ 4 minutes - testsuite-summary-speed.sh: 5 runs, ~ 7 minutes Signed-off-by: Stefano Lattarini <stefano.lattar...@gmail.com> --- t/list-of-tests.mk | 2 + t/testsuite-recheck-speed.sh | 99 ++++++++++++++++++++++++++++++++++++++++++ t/testsuite-summary-speed.sh | 76 ++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100755 t/testsuite-recheck-speed.sh create mode 100755 t/testsuite-summary-speed.sh diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index fef7165..2830073 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -775,6 +775,8 @@ t/testsuite-summary-color.sh \ t/testsuite-summary-count.sh \ t/testsuite-summary-count-many.sh \ t/testsuite-summary-reference-log.sh \ +t/testsuite-recheck-speed.sh \ +t/testsuite-summary-speed.sh \ t/test-driver-acsubst.sh \ t/test-driver-cond.sh \ t/test-driver-custom-no-extra-driver.sh \ diff --git a/t/testsuite-recheck-speed.sh b/t/testsuite-recheck-speed.sh new file mode 100755 index 0000000..9b37262 --- /dev/null +++ b/t/testsuite-recheck-speed.sh @@ -0,0 +1,99 @@ +#! /bin/sh +# Copyright (C) 2012 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Check performance of recheck target in the face of many failed tests. +# FIXME: this test is not currently able to detect whether the measured +# FIXME: performance is too low, and FAIL accordingly; it just offers an +# FIXME: easy way to verify how effective a performance optimization is. + +am_parallel_tests=yes +. ./defs || Exit 1 + +count=5000 + +cat >> configure.ac <<'END' +AC_OUTPUT +END + +cat > Makefile.am <<END +count_expected = $count +TEST_EXTENSIONS = .t +## Updated later. +TESTS = +END + +# This should ensure that our timing won't be disturbed by the time +# that would be actually required to run any of: +# - the test cases +# - the test driver executing them +# - the recipe to create the final test-suite log. +cat >> Makefile.am << 'END' +AUTOMAKE_OPTIONS = -Wno-override +## These should never be run. +T_LOG_COMPILER = false +T_LOG_DRIVER = false + +# The recipe of this also serves as a sanity check. +$(TEST_SUITE_LOG): +## For debugging. + @echo "RE-RUN:"; for i in $(TEST_LOGS); do echo " $$i"; done +## All the test cases should have been re-run. + @count_got=`for i in $(TEST_LOGS); do echo $$i; done | wc -l` \ + && echo "Count expected: $(count_expected)" \ + && echo "Count obtained: $$count_got" \ + && test $$count_got -eq $(count_expected) +## Pre-existing log files of the tests to re-run should have been +## removed by the 'recheck' target + @for i in $(TEST_LOGS); do \ + test ! -f $$i.log || { echo "$$i.log exists!"; exit 1; }; \ + done +## Actually create the target file, for extra safety. + @echo dummy > $@ +END + +# Updated later. +: > all + +# Temporarily disable shell traces, to avoid bloating the log file. +set +x + +for i in `seq_ 1 $count`; do + echo dummy $i > $i.log + echo :global-test-result: PASS > $i.trs + echo :test-result: PASS >> $i.trs + echo :recheck: yes >> $i.trs + echo TESTS += $i.t >> Makefile.am + echo $i >> all +done + +# Re-enable shell traces. +set -x + +# So that we don't need to create a ton of dummy tests. +echo '$(TESTS):' >> Makefile.am + +head -n 100 Makefile.am || : # For debugging. +tail -n 100 Makefile.am || : # Likewise. +cat $count.trs # Likewise, just the last specimen though. + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure +$MAKE recheck + +: diff --git a/t/testsuite-summary-speed.sh b/t/testsuite-summary-speed.sh new file mode 100755 index 0000000..4d18c61 --- /dev/null +++ b/t/testsuite-summary-speed.sh @@ -0,0 +1,76 @@ +#! /bin/sh +# Copyright (C) 2012 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Check performanceof recipe generating test-suite.log file and testsuite +# summary. That has suffered of huge inefficiencies in the past. +# FIXME: this test is not currently able to detect whether the measured +# FIXME: performance is too low, and FAIL accordingly; it just offers +# FIXME: an easy way verify how effective a performance optimization is. + +am_parallel_tests=yes +. ./defs || Exit 1 + +count=10000 + +echo AC_OUTPUT >> configure.ac + +cat > Makefile.am <<'END' +TEST_EXTENSIONS = .t +## Will be updated later. +TESTS = +## None of these should ever be run. +T_LOG_COMPILER = false +T_LOG_DRIVER = false +END + +# Temporarily disable shell traces, to avoid bloating the log file. +set +x + +for i in `seq_ 1 $count`; do + echo false > $i.t + echo dummy $i > $i.log + echo :global-test-result: PASS > $i.trs + echo :test-result: PASS >> $i.trs + echo :copy-in-global-log: yes >> $i.trs + echo TESTS += $i.t +done >> Makefile.am + +# Re-enable shell traces. +set -x + +head -n 100 Makefile.am || : # For debugging. +tail -n 100 Makefile.am || : # Likewise. +cat $count.trs # Likewise, just the last specimen though. + +# So that we don't need to create a ton of dummy tests. +#echo '$(TESTS):' >> Makefile.am + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure + +$MAKE test-suite.log >stdout || { cat stdout; Exit 1; } +cat stdout + +# Sanity checks. +grep "^# TOTAL: $count$" stdout +grep "^dummy $count$" test-suite.log +specimen=347 # Could be any number <= $count. +grep "^dummy $specimen$" test-suite.log + +: -- 1.7.9.5