On Thu, Nov 12, 2015 at 4:43 PM, <[email protected]> wrote: > From: Dylan Baker <[email protected]> > > This changes the behavior of the dEQP integration such that a status > that is < 0 on unix or (< 0 || == 3) on windows will be a crash rather > than a fail. A status > 0 (except 3 on windows) will still be marked > fail. > > This makes use of the helper function from framework/test/base.py > rather than calling super() (which would still call the helper) because > we don't want to get the warn status that we would also inherit. > > v2: - Update tests to use actual deqp ouput. > - Fix status loop, v1 had a bug that would cause the loop to not > exit when it needed to, but it would pass the simplified tests.
Seems to work as advertised this time. Thanks! Tested-by: Jason Ekstrand <[email protected]> > cc: Jason Ekstrand <[email protected]> > Signed-off-by: Dylan Baker <[email protected]> > --- > framework/test/base.py | 5 +- > framework/test/deqp.py | 23 +++-- > framework/tests/deqp_tests.py | 191 > ++++++++++++++++++++++++++++++++++-------- > 3 files changed, 174 insertions(+), 45 deletions(-) > > diff --git a/framework/test/base.py b/framework/test/base.py > index bf00396..e130ec5 100644 > --- a/framework/test/base.py > +++ b/framework/test/base.py > @@ -47,6 +47,7 @@ __all__ = [ > 'TestRunError', > 'ValgrindMixin', > 'WindowResizeMixin', > + 'is_crash_returncode', > ] > > > @@ -112,7 +113,7 @@ class ProcessTimeout(threading.Thread): > return self.status > > > -def _is_crash_returncode(returncode): > +def is_crash_returncode(returncode): > """Determine whether the given process return code correspond to a > crash. > """ > @@ -204,7 +205,7 @@ class Test(object): > def interpret_result(self): > """Convert the raw output of the test into a form piglit understands. > """ > - if _is_crash_returncode(self.result.returncode): > + if is_crash_returncode(self.result.returncode): > # check if the process was terminated by the timeout > if self.timeout > 0 and self.__proc_timeout.join() > 0: > self.result.result = 'timeout' > diff --git a/framework/test/deqp.py b/framework/test/deqp.py > index 8290faf..5c84131 100644 > --- a/framework/test/deqp.py > +++ b/framework/test/deqp.py > @@ -24,7 +24,8 @@ import subprocess > > # Piglit modules > from framework import core, grouptools, exceptions > -from framework.profile import Test, TestProfile > +from framework.profile import TestProfile > +from framework.test.base import Test, is_crash_returncode > > __all__ = [ > 'DEQPBaseTest', > @@ -141,11 +142,10 @@ class DEQPBaseTest(Test): > command = super(DEQPBaseTest, self).command > return command + self.extra_args > > - def interpret_result(self): > - if self.result.returncode != 0: > - self.result.result = 'fail' > - return > - > + def __find_map(self): > + """Run over the lines and set the result.""" > + # splitting this into a separate function allows us to return > cleanly, > + # otherwise this requires some break/else/continue madness > for line in self.result.out.split('\n'): > line = line.lstrip() > for k, v in self.__RESULT_MAP.iteritems(): > @@ -153,5 +153,14 @@ class DEQPBaseTest(Test): > self.result.result = v > return > > + def interpret_result(self): > + if is_crash_returncode(self.result.returncode): > + self.result.result = 'crash' > + elif self.result.returncode != 0: > + self.result.result = 'fail' > + else: > + self.__find_map() > + > # We failed to parse the test output. Fallback to 'fail'. > - self.result.result = 'fail' > + if self.result.result == 'notrun': > + self.result.result = 'fail' > diff --git a/framework/tests/deqp_tests.py b/framework/tests/deqp_tests.py > index d9327ec..d95390d 100644 > --- a/framework/tests/deqp_tests.py > +++ b/framework/tests/deqp_tests.py > @@ -25,6 +25,9 @@ tests > > """ > > +from __future__ import absolute_import, division, print_function > + > +import mock > import nose.tools as nt > > from framework import profile, grouptools, exceptions > @@ -137,42 +140,158 @@ def test_DEQPBaseTest_command(): > nt.eq_(test.command[-1], 'extra') > > > -def test_DEQPBaseTest_interpret_result_returncode(): > - """deqp.DEQPBaseTest.interpret_result: if returncode is not 0 result is > fail > - """ > - test = _DEQPTestTest('a.deqp.test') > - test.result.returncode = 1 > - test.interpret_result() > - > - nt.eq_(test.result.result, 'fail') > +class TestDEQPBaseTestInterpretResult(object): > + """Tests for DEQPBaseTest.interpret_result. > > + This specifically tests the part that searches stdout. > > -def test_DEQPBaseTest_interpret_result_fallthrough(): > - """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail > """ > - test = _DEQPTestTest('a.deqp.test') > - test.result.returncode = 0 > - test.result.out = '' > - test.interpret_result() > - > - nt.eq_(test.result.result, 'fail') > - > - > [email protected]_generator > -def test_DEQPBaseTest_interpret_result_status(): > - """generate tests for each status possiblility.""" > - def test(status, expected): > - inst = _DEQPTestTest('a.deqp.test') > - inst.result.returncode = 0 > - inst.result.out = status > - inst.interpret_result() > - nt.eq_(inst.result.result, expected) > - > - desc = ('deqp.DEQPBaseTest.interpret_result: ' > - 'when "{}" in stdout status is set to "{}"') > - > - _map = deqp.DEQPBaseTest._DEQPBaseTest__RESULT_MAP.iteritems() # > pylint: disable=no-member,protected-access > - > - for status, expected in _map: > - test.description = desc.format(status, expected) > - yield test, status, expected > + def __init__(self): > + self.test = None > + > + def setup(self): > + self.test = _DEQPTestTest('a.deqp.test') > + > + def test_crash(self): > + """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts > is crash""" > + self.test.result.returncode = -9 > + self.test.interpret_result() > + nt.eq_(self.test.result.result, 'crash') > + > + def test_returncode_fail(self): > + """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result > is fail""" > + self.test.result.returncode = 1 > + self.test.interpret_result() > + nt.eq_(self.test.result.result, 'fail') > + > + def test_fallthrough(self): > + """deqp.DEQPBaseTest.interpret_result: if no case is hit set to > fail""" > + self.test.result.returncode = 0 > + self.test.result.out = '' > + self.test.interpret_result() > + nt.eq_(self.test.result.result, 'fail') > + > + def test_windows_returncode_3(self): > + """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is > crash""" > + self.test.result.returncode = 3 > + with mock.patch('framework.test.base.sys.platform', 'win32'): > + self.test.interpret_result() > + nt.eq_(self.test.result.result, 'crash') > + > + > +class TestDEQPBaseTestIntepretResultStatus(object): > + """Tests for DEQPBaseTest.__find_map.""" > + def __init__(self): > + self.inst = None > + > + @staticmethod > + def __make_out(status): Did you seriously just name a function "make out"? How about "generate_test_output" or "dummy_deqp_output" or something like that? > + out = ( > + "dEQP Core 2014.x (0xcafebabe) starting..\n" > + "arget implementation = 'DRM'\n" > + "\n" > + "Test case > 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..\n" > + "Vertex shader compile time = 0.129000 ms\n" > + "Fragment shader compile time = 0.264000 ms\n" > + "Link time = 0.814000 ms\n" > + "Test case duration in microseconds = 487155 us\n" > + "{stat} ({stat})\n" > + "\n" > + "DONE!\n" > + "\n" > + "Test run totals:\n" > + "Passed: {pass_}/1 (100.0%)\n" > + "Failed: {fail}/1 (0.0%)\n" > + "Not supported: {supp}/1 (0.0%)\n" > + "Warnings: {warn}/1 (0.0%)\n" > + ) > + > + if status == 'Fail': > + return out.format( > + stat=status, > + pass_=0, > + fail=1, > + supp=0, > + warn=0, > + ) > + elif status == 'NotSupported': > + return out.format( > + stat=status, > + pass_=0, > + fail=0, > + supp=0, > + warn=0, > + ) > + elif status == 'Pass': > + return out.format( > + stat=status, > + pass_=1, > + fail=0, > + supp=0, > + warn=0, > + ) > + elif status == 'QualityWarning': > + return out.format( > + stat=status, > + pass_=0, > + fail=0, > + supp=0, > + warn=1, > + ) > + elif status == 'InternalError': > + return out.format( > + stat=status, > + pass_=0, > + fail=0, > + supp=1, > + warn=0, > + ) > + elif status == 'Crash': > + return out.format( > + stat=status, > + pass_=0, > + fail=1, > + supp=0, > + warn=0, > + ) > + raise Exception('Unreachable') > + > + def setup(self): > + self.inst = _DEQPTestTest('a.deqp.test') > + self.inst.result.returncode = 0 > + > + def test_fail(self): > + """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the > result is 'fail'""" > + self.inst.result.out = self.__make_out('Fail') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'fail') > + > + def test_pass(self): > + """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the > result is 'Pass'""" > + self.inst.result.out = self.__make_out('Pass') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'pass') > + > + def test_warn(self): > + """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in > result the result is 'warn'""" > + self.inst.result.out = self.__make_out('QualityWarning') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'warn') > + > + def test_error(self): > + """test.deqp.DEQPBaseTest.interpret_result: when InternalError in > result the result is 'fail'""" > + self.inst.result.out = self.__make_out('InternalError') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'fail') > + > + def test_crash(self): > + """test.deqp.DEQPBaseTest.interpret_result: when InternalError in > result the result is 'crash'""" > + self.inst.result.out = self.__make_out('Crash') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'crash') > + > + def test_skip(self): > + """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in > result the result is 'skip'""" > + self.inst.result.out = self.__make_out('NotSupported') > + self.inst.interpret_result() > + nt.eq_(self.inst.result.result, 'skip') > -- > 2.6.2 > _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
