Review at https://gerrit.osmocom.org/2669
Improve test reporting code * Add Junit output file support * Differentiate between an expected failure test and an error in the test, as described in JUnit. * In case of an error/exception during test, record and attach it to the Test object and continue running the tests, and show it at the end during the trial report. Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5 --- M selftest/suite_test.ok M selftest/suite_test.ok.ign M selftest/suite_test.py A selftest/suite_test/test_suite/test_fail.py A selftest/suite_test/test_suite/test_fail_raise.py M src/osmo-gsm-tester.py M src/osmo_gsm_tester/suite.py M src/osmo_gsm_tester/test.py A src/osmo_gsm_tester/trial_report.py A suites/debug/error.py M suites/debug/fail.py A suites/debug/fail_raise.py A suites/debug/pass.py 13 files changed, 325 insertions(+), 68 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/69/2669/1 diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok index fda77dc..8c9daef 100644 --- a/selftest/suite_test.ok +++ b/selftest/suite_test.ok @@ -59,15 +59,79 @@ tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py] -pass: all 1 tests passed. +pass: all 6 tests passed (5 skipped). - a test with an error tst test_suite: Suite run start [suite.py:[LINENR]] tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]] tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]] -tst test_error.py:[LINENR]: FAIL [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]] -tst test_error.py:[LINENR]: ERR: AssertionError: [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False] -FAIL: 1 of 1 tests failed: - test_error.py +tst test_error.py:[LINENR]: ERR: AssertionError: [test_error.py:[LINENR]: assert False] +tst test_error.py:[LINENR] ERROR (AssertionError) [test_suite↪test_error.py] [suite.py:[LINENR]] +FAIL: [test_suite] failed->0 error->1 out of 6 tests run: + SKIP: [hello_world.py] + SKIP: [mo_mt_sms.py] + SKIP: [mo_sms.py] + ERROR: [test_error.py] ([TS_ISO8601], 0 sec) type:'AssertionError' message: AssertionError() + Traceback (most recent call last): + File "[PATH]/selftest/../src/osmo_gsm_tester/suite.py", line [LINENR], in run + self.path) + File "[PATH]/selftest/../src/osmo_gsm_tester/util.py", line [LINENR], in run_python_file + spec.loader.exec_module( importlib.util.module_from_spec(spec) ) + File "<frozen importlib._bootstrap_external>", line [LINENR], in exec_module + File "<frozen importlib._bootstrap>", line [LINENR], in _call_with_frames_removed + File "[PATH]/selftest/suite_test/test_suite/test_error.py", line [LINENR], in <module> + assert False + AssertionError + SKIP: [test_fail.py] + SKIP: [test_fail_raise.py] + +- a test with a failure +tst test_suite: Suite run start [suite.py:[LINENR]] +tst test_fail.py:[LINENR] START [test_suite↪test_fail.py] [suite.py:[LINENR]] +tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]] +tst test_fail.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail.py] [suite.py:[LINENR]] +FAIL: [test_suite] failed->1 error->0 out of 6 tests run: + SKIP: [hello_world.py] + SKIP: [mo_mt_sms.py] + SKIP: [mo_sms.py] + SKIP: [test_error.py] + FAIL: [test_fail.py] ([TS_ISO8601], 0 sec) type:'EpicFail' message: This failure is expected + File "[PATH]/selftest/suite_test.py", line [LINENR], in <module> + results = s.run_tests('test_fail.py') + File "[PATH]/selftest/../src/osmo_gsm_tester/suite.py", line [LINENR], in run_tests + test.run(self) + File "[PATH]/selftest/../src/osmo_gsm_tester/suite.py", line [LINENR], in run + self.path) + File "[PATH]/selftest/../src/osmo_gsm_tester/util.py", line [LINENR], in run_python_file + spec.loader.exec_module( importlib.util.module_from_spec(spec) ) + File "<frozen importlib._bootstrap_external>", line [LINENR], in exec_module + File "<frozen importlib._bootstrap>", line [LINENR], in _call_with_frames_removed + File "[PATH]/selftest/suite_test/test_suite/test_fail.py", line [LINENR], in <module> + test.set_fail('EpicFail', 'This failure is expected') + SKIP: [test_fail_raise.py] + +- a test with a raised failure +tst test_suite: Suite run start [suite.py:[LINENR]] +tst test_fail_raise.py:[LINENR] START [test_suite↪test_fail_raise.py] [suite.py:[LINENR]] +tst test_fail_raise.py:[LINENR]: I am 'test_suite' / 'test_fail_raise.py:[LINENR]' [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]] +tst test_fail_raise.py:[LINENR]: ERR: Failure: ('EpicFail', 'This failure is expected') [test_fail_raise.py:[LINENR]: raise Failure('EpicFail', 'This failure is expected')] +tst test_fail_raise.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail_raise.py] [suite.py:[LINENR]] +FAIL: [test_suite] failed->1 error->0 out of 6 tests run: + SKIP: [hello_world.py] + SKIP: [mo_mt_sms.py] + SKIP: [mo_sms.py] + SKIP: [test_error.py] + SKIP: [test_fail.py] + FAIL: [test_fail_raise.py] ([TS_ISO8601], 0 sec) type:'EpicFail' message: This failure is expected + Traceback (most recent call last): + File "[PATH]/selftest/../src/osmo_gsm_tester/suite.py", line [LINENR], in run + self.path) + File "[PATH]/selftest/../src/osmo_gsm_tester/util.py", line [LINENR], in run_python_file + spec.loader.exec_module( importlib.util.module_from_spec(spec) ) + File "<frozen importlib._bootstrap_external>", line [LINENR], in exec_module + File "<frozen importlib._bootstrap>", line [LINENR], in _call_with_frames_removed + File "[PATH]/selftest/suite_test/test_suite/test_fail_raise.py", line [LINENR], in <module> + raise Failure('EpicFail', 'This failure is expected') + osmo_gsm_tester.suite.Failure: ('EpicFail', 'This failure is expected') - graceful exit. diff --git a/selftest/suite_test.ok.ign b/selftest/suite_test.ok.ign index a19fb8b..20d6c93 100644 --- a/selftest/suite_test.ok.ign +++ b/selftest/suite_test.ok.ign @@ -1,2 +1,4 @@ /[^ ]*/selftest/ [PATH]/selftest/ \.py:[0-9]* .py:[LINENR] +[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2} [TS_ISO8601] +,\ line\ [0-9]*, , line [LINENR], diff --git a/selftest/suite_test.py b/selftest/suite_test.py index 315c683..548b966 100755 --- a/selftest/suite_test.py +++ b/selftest/suite_test.py @@ -30,5 +30,13 @@ results = s.run_tests('test_error.py') print(str(results)) +print('\n- a test with a failure') +results = s.run_tests('test_fail.py') +print(str(results)) + +print('\n- a test with a raised failure') +results = s.run_tests('test_fail_raise.py') +print(str(results)) + print('\n- graceful exit.') # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/selftest/suite_test/test_suite/test_fail.py b/selftest/suite_test/test_suite/test_fail.py new file mode 100755 index 0000000..6880c81 --- /dev/null +++ b/selftest/suite_test/test_suite/test_fail.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +from osmo_gsm_tester.test import * + +print('I am %r / %r' % (suite.name(), test.name())) + +test.set_fail('EpicFail', 'This failure is expected') diff --git a/selftest/suite_test/test_suite/test_fail_raise.py b/selftest/suite_test/test_suite/test_fail_raise.py new file mode 100755 index 0000000..a7b0b61 --- /dev/null +++ b/selftest/suite_test/test_suite/test_fail_raise.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +from osmo_gsm_tester.test import * + +print('I am %r / %r' % (suite.name(), test.name())) + +raise Failure('EpicFail', 'This failure is expected') diff --git a/src/osmo-gsm-tester.py b/src/osmo-gsm-tester.py index 0a04708..ddec926 100755 --- a/src/osmo-gsm-tester.py +++ b/src/osmo-gsm-tester.py @@ -69,7 +69,7 @@ import sys import argparse from osmo_gsm_tester import __version__ -from osmo_gsm_tester import trial, suite, log, config +from osmo_gsm_tester import trial, suite, log, config, trial_report def main(): @@ -169,44 +169,45 @@ t.verify() trials.append(t) - trials_passed = [] - trials_failed = [] + trials_run = [] + any_failed = False for current_trial in trials: try: with current_trial: - suites_passed = [] - suites_failed = [] + trial_failed = False + report = trial_report.TrialReport(current_trial) + for suite_scenario_str, suite_def, scenarios in suite_scenarios: log.large_separator(current_trial.name(), suite_scenario_str) suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios) result = suite_run.run_tests(test_names) + report.add_results(result) if result.all_passed: - suites_passed.append(suite_scenario_str) suite_run.log('PASS') else: - suites_failed.append(suite_scenario_str) + trial_failed = True suite_run.err('FAIL') - if not suites_failed: + + if not trial_failed: current_trial.log('PASS') - trials_passed.append(current_trial.name()) else: current_trial.err('FAIL') - trials_failed.append((current_trial.name(), suites_passed, suites_failed)) + any_failed = True + report.write_junit_report() + trials_run.append((current_trial.name(), report)) except: current_trial.log_exn() sys.stderr.flush() sys.stdout.flush() log.large_separator() - if trials_passed: - print('Trials passed:\n ' + ('\n '.join(trials_passed))) - if trials_failed: - print('Trials failed:') - for trial_name, suites_passed, suites_failed in trials_failed: - print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed))) - for suite_failed in suites_failed: - print(' FAIL:', suite_failed) + if not any_failed: + print('All trials passed:\n ' + ('\n '.join(trial_name for trial_name, report in trials_run))) + else: + for trial_name, report in trials_run: + log.large_separator('Trial Report for %s' % trial_name) + report.log_report() exit(1) if __name__ == '__main__': diff --git a/src/osmo_gsm_tester/suite.py b/src/osmo_gsm_tester/suite.py index 43e55af..4c99c5e 100644 --- a/src/osmo_gsm_tester/suite.py +++ b/src/osmo_gsm_tester/suite.py @@ -21,11 +21,19 @@ import sys import time import copy +import traceback +import xml.etree.ElementTree as et +from datetime import datetime from . import config, log, template, util, resource, schema, ofono_client, osmo_nitb from . import test class Timeout(Exception): pass + +class Failure(Exception): + def __init__(self, fail_type='', fail_msg=''): + self.fail_type = fail_type + self.fail_msg = fail_msg class SuiteDefinition(log.Origin): '''A test suite reserves resources for a number of tests. @@ -78,9 +86,16 @@ raise ValueError('add_test(): test already belongs to another suite') self.tests.append(test) - - class Test(log.Origin): + UNKNOWN = 0 + SKIP = 1 + PASS = 2 + FAIL = 3 + ERROR = 4 + @staticmethod + def status2str(st): + stats = ['UNKNOWN', 'SKIP', 'PASS', 'FAIL', 'ERROR'] + return stats[st] def __init__(self, suite, test_basename): self.suite = suite @@ -89,32 +104,95 @@ super().__init__(self.path) self.set_name(self.basename) self.set_log_category(log.C_TST) + self.status = Test.UNKNOWN + self.ts_start = 0 + self.ts_end = 0 + self.time = 0 + self.fail_type = '' + self.fail_message = '' def run(self, suite_run): assert self.suite is suite_run.definition - with self: - test.setup(suite_run, self, ofono_client, sys.modules[__name__]) - success = False - try: + try: + with self: + self.status = Test.UNKNOWN + self.ts_start = time.time() + test.setup(suite_run, self, ofono_client, sys.modules[__name__]) self.log('START') with self.redirect_stdout(): util.run_python_file('%s.%s' % (self.suite.name(), self.name()), self.path) - success = True - except resource.NoResourceExn: - self.err('Current resource state:\n', repr(suite_run.reserved_resources)) - raise - finally: - if success: - self.log('PASS') + if self.status == Test.UNKNOWN: + self.set_pass() + except Exception as e: + self.log_exn() + if self.status == Test.UNKNOWN: + if isinstance(e, Failure): + self.set_fail(e.fail_type, e.fail_msg + '\n' + traceback.format_exc().rstrip(), False) else: - self.log('FAIL') + msg = repr(e) + '\n' + traceback.format_exc().rstrip() + if isinstance(e, resource.NoResourceExn): + msg += '\n' + 'Current resource state:\n' + repr(suite_run.reserved_resources) + self.set_error(type(e).__name__, msg, False) + + finally: + if self.status == Test.PASS or self.status == Test.SKIP: + self.log(Test.status2str(self.status)) + else: + self.log('%s (%s)' % (Test.status2str(self.status), self.fail_type)) def name(self): l = log.get_line_for_src(self.path) if l is not None: return '%s:%s' % (self._name, l) return super().name() + + def end_with_status(self, st, fail_type='', fail_message='', tb=False): + self.status = st + self.ts_end = time.time() + self.time = round(self.ts_end - self.ts_start) + self.fail_type = fail_type + self.fail_message = fail_message + if tb: + self.fail_message += '\n' + ''.join(traceback.format_stack()[:-2]).rstrip() + + def set_fail(self, fail_type='', fail_message='', tb=True): + self.end_with_status(Test.FAIL, fail_type, fail_message, tb) + + def set_error(self, fail_type='', fail_message='', tb=True): + self.end_with_status(Test.ERROR, fail_type, fail_message, tb) + + def set_pass(self): + self.end_with_status(Test.PASS) + + def set_skip(self): + self.status = Test.SKIP + self.ts_end = time.time() + self.time = 0 + + def to_junit(self): + testcase = et.Element('testcase') + testcase.set('name', self.name()) + testcase.set('time', str(self.time)) + if self.status == Test.SKIP: + skip = et.SubElement(testcase, 'skipped') + elif self.status == Test.FAIL: + failure = et.SubElement(testcase, 'failure') + failure.set('type', self.fail_type) + failure.text = self.fail_message + elif self.status == Test.ERROR: + failure = et.SubElement(testcase, 'error') + failure.set('type', self.fail_type) + failure.text = self.fail_message + return testcase + + def __str__(self): + ret = "%s: [%s]" % (Test.status2str(self.status), self.name()) + if self.status != Test.SKIP: + ret += " (%s, %d sec)" % (datetime.fromtimestamp(round(self.ts_start)).isoformat(), self.time) + if self.status == Test.FAIL or self.status == Test.ERROR: + ret += " type:'%s' message: %s" % (self.fail_type, self.fail_message.replace('\n', '\n ')) + return ret class SuiteRun(log.Origin): @@ -132,6 +210,7 @@ self.set_name(suite_scenario_str) self.set_log_category(log.C_TST) self.resources_pool = resource.ResourcesPool() + self.tests = [] def combined(self, conf_name): self.dbg(combining=conf_name) @@ -158,30 +237,55 @@ return self._config class Results: - def __init__(self): - self.passed = [] - self.failed = [] - self.all_passed = None + def __init__(self, suite_run): + self.suite_run = suite_run + self.tests = [] + self.all_passed = True + self.ts_start = time.time() + self.ts_end = time.time() + self.time = 0 + self.failed_ctr = 0 + self.error_ctr = 0 + self.skipped_ctr = 0 - def add_pass(self, test): - self.passed.append(test) - - def add_fail(self, test): - self.failed.append(test) + def add_test(self, test): + self.tests.append(test) + if test.status == Test.FAIL: + self.failed_ctr += 1 + self.all_passed = False + if test.status == Test.ERROR: + self.error_ctr += 1 + self.all_passed = False + if test.status == Test.SKIP: + self.skipped_ctr += 1 def conclude(self): - self.all_passed = bool(self.passed) and not bool(self.failed) + self.ts_end = time.time() + self.time = round(self.ts_end - self.ts_start) return self + def to_junit(self): + testsuite = et.Element('testsuite') + testsuite.set('name', self.suite_run.name()) + testsuite.set('hostname', 'localhost') + testsuite.set('timestamp', datetime.fromtimestamp(round(self.ts_start)).isoformat()) + testsuite.set('time', str(self.time)) + testsuite.set('tests', str(len(self.tests))) + testsuite.set('failures', str(self.failed_ctr)) + testsuite.set('errors', str(self.error_ctr)) + for test in self.tests: + testcase = test.to_junit() + testsuite.append(testcase) + return testsuite + def __str__(self): - if self.failed: - return 'FAIL: %d of %d tests failed:\n %s' % ( - len(self.failed), - len(self.failed) + len(self.passed), - '\n '.join([t.name() for t in self.failed])) - if not self.passed: + if self.failed_ctr or self.error_ctr: + return 'FAIL: [%s] failed->%d error->%d out of %d tests run:\n %s' % ( + self.suite_run.name(), self.failed_ctr, self.error_ctr, len(self.tests), + '\n '.join([str(t) for t in self.tests])) + if not self.tests: return 'no tests were run.' - return 'pass: all %d tests passed.' % len(self.passed) + return 'pass: all %d tests passed (%d skipped).' % (len(self.tests), self.skipped_ctr) def reserve_resources(self): if self.reserved_resources: @@ -194,22 +298,16 @@ self.log('Suite run start') if not self.reserved_resources: self.reserve_resources() - results = SuiteRun.Results() + results = SuiteRun.Results(self) for test in self.definition.tests: if names and not test.name() in names: - continue - self._run_test(test, results) + test.set_skip() + else: + with self: + test.run(self) + results.add_test(test) self.stop_processes() return results.conclude() - - def _run_test(self, test, results): - try: - with self: - test.run(self) - results.add_pass(test) - except: - results.add_fail(test) - self.log_exn() def remember_to_stop(self, process): if self._processes is None: diff --git a/src/osmo_gsm_tester/test.py b/src/osmo_gsm_tester/test.py index 871e3ae..f584c92 100644 --- a/src/osmo_gsm_tester/test.py +++ b/src/osmo_gsm_tester/test.py @@ -32,9 +32,10 @@ poll = None prompt = None Timeout = None +Failure = None def setup(suite_run, _test, ofono_client, suite_module): - global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Timeout + global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Failure, Timeout trial = suite_run.trial suite = suite_run test = _test @@ -46,6 +47,7 @@ sleep = suite_run.sleep poll = suite_run.poll prompt = suite_run.prompt + Failure = suite_module.Failure Timeout = suite_module.Timeout # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/src/osmo_gsm_tester/trial_report.py b/src/osmo_gsm_tester/trial_report.py new file mode 100644 index 0000000..641b5dc --- /dev/null +++ b/src/osmo_gsm_tester/trial_report.py @@ -0,0 +1,55 @@ +# osmo_gsm_tester: trial: directory of binaries to be tested +# +# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH +# +# Author: Pau Espin Pedrol <pes...@sysmocom.de> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from . import log, util +import xml.etree.ElementTree as et +import os + +class TrialReport(log.Origin): + + def __init__(self, trial): + self.trial_name = trial.name() + self.results = [] + self.filepath = trial.get_run_dir().new_file(self.trial_name+'.xml') + + def add_results(self, results): + self.results.append(results) + + def to_junit(self): + testsuites = et.Element('testsuites') + for result in self.results: + testsuite = result.to_junit() + testsuites.append(testsuite) + return testsuites + + def write_junit_report(self): + self.log("Storing JUnit report in ", self.filepath) + elements = et.ElementTree(element=self.to_junit()) + elements.write(self.filepath) + + def log_report(self): + failed = False + for result in self.results: + if not result.all_passed: + failed = True + + msg = '\n%s [%s]\n ' % ('FAIL' if failed else 'PASS', self.trial_name) + msg += '\n '.join(str(result) for result in self.results) + self.log(msg) +# vim: expandtab tabstop=4 shiftwidth=4 diff --git a/suites/debug/error.py b/suites/debug/error.py new file mode 100644 index 0000000..8e146fa --- /dev/null +++ b/suites/debug/error.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from osmo_gsm_tester.test import * + +# This can be used to verify that a test error is reported properly. +assert False diff --git a/suites/debug/fail.py b/suites/debug/fail.py index 1b412b5..fcd56e0 100644 --- a/suites/debug/fail.py +++ b/suites/debug/fail.py @@ -2,4 +2,4 @@ from osmo_gsm_tester.test import * # This can be used to verify that a test failure is reported properly. -assert False +test.set_fail('EpicFail', 'This failure is expected') diff --git a/suites/debug/fail_raise.py b/suites/debug/fail_raise.py new file mode 100644 index 0000000..22fb940 --- /dev/null +++ b/suites/debug/fail_raise.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from osmo_gsm_tester.test import * + +# This can be used to verify that a test failure is reported properly. +raise Failure('EpicFail', 'This failure is expected') diff --git a/suites/debug/pass.py b/suites/debug/pass.py new file mode 100644 index 0000000..4deb02e --- /dev/null +++ b/suites/debug/pass.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from osmo_gsm_tester.test import * + +# This can be used to verify that a test passes correctly. +assert True -- To view, visit https://gerrit.osmocom.org/2669 To unsubscribe, visit https://gerrit.osmocom.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5 Gerrit-PatchSet: 1 Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-Owner: Pau Espin Pedrol <pes...@sysmocom.de>