Re: [U-Boot] [PATCH 6/9] test/py: correctly log xfail/xpass tests

[Simon Glass]

On 28 January 2016 at 20:47, Simon Glass  wrote:
> On 27 January 2016 at 23:57, Stephen Warren  wrote:
>> From: Stephen Warren 
>>
>> Tests can complete in passed, skipped, xpass, xfailed, or failed, states.
>> Currently the U-Boot log generation code doesn't handle the xfailed or
>> xpass states since they aren't used. Add support for the remaining states.
>> Without this, tests that xfail end up being reported as skipped.
>>
>> Signed-off-by: Stephen Warren 
>> ---
>>  test/py/conftest.py | 60 
>> +
>>  test/py/multiplexed_log.css |  8 ++
>>  test/py/multiplexed_log.py  | 30 ---
>>  3 files changed, 74 insertions(+), 24 deletions(-)
>
> Acked-by: Simon Glass 

Applied to u-boot-dm, thanks!
___
U-Boot mailing list
U-Boot@lists.denx.de
http://lists.denx.de/mailman/listinfo/u-boot

Re: [U-Boot] [PATCH 6/9] test/py: correctly log xfail/xpass tests

[Simon Glass]

On 27 January 2016 at 23:57, Stephen Warren  wrote:
> From: Stephen Warren 
>
> Tests can complete in passed, skipped, xpass, xfailed, or failed, states.
> Currently the U-Boot log generation code doesn't handle the xfailed or
> xpass states since they aren't used. Add support for the remaining states.
> Without this, tests that xfail end up being reported as skipped.
>
> Signed-off-by: Stephen Warren 
> ---
>  test/py/conftest.py | 60 
> +
>  test/py/multiplexed_log.css |  8 ++
>  test/py/multiplexed_log.py  | 30 ---
>  3 files changed, 74 insertions(+), 24 deletions(-)

Acked-by: Simon Glass 
___
U-Boot mailing list
U-Boot@lists.denx.de
http://lists.denx.de/mailman/listinfo/u-boot

[U-Boot] [PATCH 6/9] test/py: correctly log xfail/xpass tests

[Stephen Warren]

From: Stephen Warren 

Tests can complete in passed, skipped, xpass, xfailed, or failed, states.
Currently the U-Boot log generation code doesn't handle the xfailed or
xpass states since they aren't used. Add support for the remaining states.
Without this, tests that xfail end up being reported as skipped.

Signed-off-by: Stephen Warren 
---
 test/py/conftest.py | 60 +
 test/py/multiplexed_log.css |  8 ++
 test/py/multiplexed_log.py  | 30 ---
 3 files changed, 74 insertions(+), 24 deletions(-)

diff --git a/test/py/conftest.py b/test/py/conftest.py
index 9c9426aebe10..3e162cafcc4a 100644
--- a/test/py/conftest.py
+++ b/test/py/conftest.py
@@ -249,6 +249,8 @@ def u_boot_console(request):
 
 tests_not_run = set()
 tests_failed = set()
+tests_xpassed = set()
+tests_xfailed = set()
 tests_skipped = set()
 tests_passed = set()
 
@@ -289,6 +291,14 @@ def cleanup():
 log.status_skipped('%d skipped' % len(tests_skipped))
 for test in tests_skipped:
 log.status_skipped('... ' + test)
+if tests_xpassed:
+log.status_xpass('%d xpass' % len(tests_xpassed))
+for test in tests_xpassed:
+log.status_xpass('... ' + test)
+if tests_xfailed:
+log.status_xfail('%d xfail' % len(tests_xfailed))
+for test in tests_xfailed:
+log.status_xfail('... ' + test)
 if tests_failed:
 log.status_fail('%d failed' % len(tests_failed))
 for test in tests_failed:
@@ -381,34 +391,42 @@ def pytest_runtest_protocol(item, nextitem):
 """
 
 reports = runtestprotocol(item, nextitem=nextitem)
-failed = None
-skipped = None
+
+failure_cleanup = False
+test_list = tests_passed
+msg = 'OK'
+msg_log = log.status_pass
 for report in reports:
 if report.outcome == 'failed':
-failed = report
+if hasattr(report, 'wasxfail'):
+test_list = tests_xpassed
+msg = 'XPASSED'
+msg_log = log.status_xpass
+else:
+failure_cleanup = True
+test_list = tests_failed
+msg = 'FAILED:\n' + str(report.longrepr)
+msg_log = log.status_fail
 break
 if report.outcome == 'skipped':
-if not skipped:
-skipped = report
-
-if failed:
+if hasattr(report, 'wasxfail'):
+failure_cleanup = True
+test_list = tests_xfailed
+msg = 'XFAILED:\n' + str(report.longrepr)
+msg_log = log.status_xfail
+break
+test_list = tests_skipped
+msg = 'SKIPPED:\n' + str(report.longrepr)
+msg_log = log.status_skipped
+
+if failure_cleanup:
 console.drain_console()
-tests_failed.add(item.name)
-elif skipped:
-tests_skipped.add(item.name)
-else:
-tests_passed.add(item.name)
+
+test_list.add(item.name)
 tests_not_run.remove(item.name)
 
 try:
-if failed:
-msg = 'FAILED:\n' + str(failed.longrepr)
-log.status_fail(msg)
-elif skipped:
-msg = 'SKIPPED:\n' + str(skipped.longrepr)
-log.status_skipped(msg)
-else:
-log.status_pass('OK')
+msg_log(msg)
 except:
 # If something went wrong with logging, it's better to let the test
 # process continue, which may report other exceptions that triggered
@@ -424,7 +442,7 @@ def pytest_runtest_protocol(item, nextitem):
 
 log.end_section(item.name)
 
-if failed:
+if failure_cleanup:
 console.cleanup_spawn()
 
 return reports
diff --git a/test/py/multiplexed_log.css b/test/py/multiplexed_log.css
index 50f7b9092983..f6240d52da66 100644
--- a/test/py/multiplexed_log.css
+++ b/test/py/multiplexed_log.css
@@ -83,6 +83,14 @@ pre {
 color: #00
 }
 
+.status-xfail {
+color: #ff7f00
+}
+
+.status-xpass {
+color: #ff7f00
+}
+
 .status-fail {
 color: #ff
 }
diff --git a/test/py/multiplexed_log.py b/test/py/multiplexed_log.py
index fd3a9231a819..69a577e57720 100644
--- a/test/py/multiplexed_log.py
+++ b/test/py/multiplexed_log.py
@@ -408,7 +408,7 @@ class Logfile(object):
 """Write a note to the log file describing test(s) which passed.
 
 Args:
-msg: A message describing passed test(s).
+msg: A message describing the passed test(s).
 
 Returns:
 Nothing.
@@ -420,7 +420,7 @@ class Logfile(object):
 """Write a note to the log file describing skipped test(s).
 
 Args:
-msg: A message describing passed test(s).
+msg: A message describing the skipped test(s).
 
 Returns:
 Nothing.
@@ -428,11 +428,35 @@ class Logfile(object):