The branch master has been updated via 94fcec0902ec6df54a6286f61e35fa6f93815fd6 (commit) via a812549108150b53023ed62e92e8b43c9414a15e (commit) via e4522e1059a4ab32024277706f271200e0cb601b (commit) from ea4ee152a7aa022dd87f193cc6e16a7ffbfb455a (commit)
- Log ----------------------------------------------------------------- commit 94fcec0902ec6df54a6286f61e35fa6f93815fd6 Author: Dr. David von Oheimb <david.von.ohe...@siemens.com> Date: Sat Jun 27 15:45:58 2020 +0200 test/run_tests.pl: Add alias REPORT_FAILURES{,_PROGRESS} for VF and VFP Reviewed-by: Paul Dale <paul.d...@oracle.com> (Merged from https://github.com/openssl/openssl/pull/12279) commit a812549108150b53023ed62e92e8b43c9414a15e Author: Dr. David von Oheimb <david.von.ohe...@siemens.com> Date: Wed Jun 24 12:13:38 2020 +0200 test/run_tests.pl: Add visual separator after failed test case for VFP and VFP modes Reviewed-by: Paul Dale <paul.d...@oracle.com> (Merged from https://github.com/openssl/openssl/pull/12279) commit e4522e1059a4ab32024277706f271200e0cb601b Author: Dr. David von Oheimb <david.von.ohe...@siemens.com> Date: Wed Jun 24 12:12:20 2020 +0200 test/run_tests.pl: Enhance the semantics of HARNESS_VERBOSE_FAILURES (VF) Make the improved semantics of VFO replace the previous VF and remove VFO Add warnings about overriding use of HARNESS_VERBOSE* variables Reviewed-by: Paul Dale <paul.d...@oracle.com> (Merged from https://github.com/openssl/openssl/pull/12279) ----------------------------------------------------------------------- Summary of changes: INSTALL.md | 6 +++--- test/README.md | 13 +++++-------- test/run_tests.pl | 32 ++++++++++++++++++-------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3b993585d2..6989410e87 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1603,9 +1603,9 @@ that isn't a problem in OpenSSL itself (like an OS malfunction or a Perl issue). You may want increased verbosity, that can be accomplished as described in section [Test Failures of test/README.md](test/README.md#test-failures). -You may want to selectively specify which test(s) to perform. This can be done -sing the `make` variable `TESTS` as described in section [Running Selected Tests -of test/README.md](test/README.md#running-selected-tests). +You may also want to selectively specify which test(s) to perform. This can be +done using the `make` variable `TESTS` as described in section [Running +Selected Tests of test/README.md](test/README.md#running-selected-tests). If you find a problem with OpenSSL itself, try removing any compiler optimization flags from the `CFLAGS` line in the Makefile and diff --git a/test/README.md b/test/README.md index 76bf1b17aa..a9e0c827d1 100644 --- a/test/README.md +++ b/test/README.md @@ -27,21 +27,18 @@ Full verbosity, showing full output of all successful and failed test cases $ mms /macro=(V=1) test ! OpenVMS $ nmake V=1 test # Windows -Verbosity on test failure (`VERBOSE_FAILURE` or `VF`, Unix example shown): +Verbosity on failed (sub-)tests only +(`VERBOSE_FAILURE` or `VF` or `REPORT_FAILURES`): $ make test VF=1 -Verbosity on failed (sub-)tests only (`VERBOSE_FAILURES_ONLY` or `VFO`): - - $ make test VFO=1 - Verbosity on failed (sub-)tests, in addition progress on succeeded (sub-)tests -(`VERBOSE_FAILURES_PROGRESS` or `VFP`): +(`VERBOSE_FAILURE_PROGRESS` or `VFP` or `REPORT_FAILURES_PROGRESS`): $ make test VFP=1 If you want to run just one or a few specific tests, you can use -the `make` variable `TESTS` to specify them, like this: +the make variable TESTS to specify them, like this: $ make TESTS='test_rsa test_dsa' test # Unix $ mms/macro="TESTS=test_rsa test_dsa" test ! OpenVMS @@ -50,7 +47,7 @@ the `make` variable `TESTS` to specify them, like this: And of course, you can combine (Unix examples shown): $ make test TESTS='test_rsa test_dsa' VF=1 - $ make test TESTS="test_cmp_*" VFO=1 + $ make test TESTS="test_cmp_*" VFP=1 You can find the list of available tests like this: diff --git a/test/run_tests.pl b/test/run_tests.pl index 5eddaf8468..d47f7cf1e6 100644 --- a/test/run_tests.pl +++ b/test/run_tests.pl @@ -9,15 +9,16 @@ use strict; use warnings; -# Recognise VERBOSE and V which is common on other projects. -# Additionally, also recognise VERBOSE_FAILURE and VF. +# Recognise VERBOSE aka V which is common on other projects. +# Additionally, recognise VERBOSE_FAILURE aka VF aka REPORT_FAILURES +# and recognise VERBOSE_FAILURE_PROGRESS aka VFP aka REPORT_FAILURES_PROGRESS. BEGIN { $ENV{HARNESS_VERBOSE} = "yes" if $ENV{VERBOSE} || $ENV{V}; - $ENV{HARNESS_VERBOSE_FAILURE} = "yes" if $ENV{VERBOSE_FAILURE} || $ENV{VF}; - $ENV{HARNESS_VERBOSE_FAILURES_ONLY} = "yes" - if $ENV{VERBOSE_FAILURES_ONLY} || $ENV{VFO}; - $ENV{HARNESS_VERBOSE_FAILURES_PROGRESS} = "yes" - if $ENV{VERBOSE_FAILURES_PROGRESS} || $ENV{VFP}; + $ENV{HARNESS_VERBOSE_FAILURE} = "yes" + if $ENV{VERBOSE_FAILURE} || $ENV{VF} || $ENV{REPORT_FAILURES}; + $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} = "yes" + if ($ENV{VERBOSE_FAILURE_PROGRESS} || $ENV{VFP} + || $ENV{REPORT_FAILURES_PROGRESS}); } use File::Spec::Functions qw/catdir catfile curdir abs2rel rel2abs/; @@ -53,9 +54,13 @@ $tapargs{jobs} = $jobs if defined $jobs; my %openssl_args = (); $openssl_args{'failure_verbosity'} = $ENV{HARNESS_VERBOSE} ? 0 : - $ENV{HARNESS_VERBOSE_FAILURE} ? 3 : - $ENV{HARNESS_VERBOSE_FAILURES_PROGRESS} ? 2 : - $ENV{HARNESS_VERBOSE_FAILURES_ONLY} ? 1 : 0; + $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} ? 2 : + 1; # $ENV{HARNESS_VERBOSE_FAILURE} +print "Warning: HARNESS_VERBOSE overrides HARNESS_VERBOSE_FAILURE*\n" + if ($ENV{HARNESS_VERBOSE} && ($ENV{HARNESS_VERBOSE_FAILURE} + || $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS})); +print "Warning: HARNESS_VERBOSE_FAILURE_PROGRESS overrides HARNESS_VERBOSE_FAILURE\n" + if ($ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} && $ENV{HARNESS_VERBOSE_FAILURE}); my $outfilename = $ENV{HARNESS_TAP_COPY}; open $openssl_args{'tap_copy'}, ">$outfilename" @@ -153,9 +158,7 @@ $eres = eval { if defined $fh; my $failure_verbosity = $openssl_args{failure_verbosity}; - if ($failure_verbosity == 3) { - push @failure_output, $self->as_string; - } elsif ($failure_verbosity > 0) { + if ($failure_verbosity > 0) { my $is_plan = $self->is_plan; my $tests_planned = $is_plan && $self->tests_planned; my $is_test = $self->is_test; @@ -180,6 +183,7 @@ $eres = eval { print $output_buffer if !$is_ok; print "\n".$self->as_string if !$is_ok || $failure_verbosity == 2; + print "\n# ------------------------------------------------------------------------------" if !$is_ok; $output_buffer = ""; } elsif ($self->as_string ne "") { # typically is_comment or is_unknown @@ -201,7 +205,7 @@ $eres = eval { print $_, "\n" foreach (("", @failure_output)); } # Echo any trailing comments etc. - print "$output_buffer" if $failure_verbosity != 3; + print "$output_buffer"; }; }