Re: [Piglit] [PATCH] framework/test/deqp: handle deqp extra args properly in more cases
Reviewed-by: Mark Janes
Dylan Baker writes:
> Currently we blindly pass deqp extra args to each test case, and only to
> the test case.
>
> There are two cases where this is problematic:
> 1) if the extra args need to be passed to the deqp command that
>generates the test list (it isn't before this patch)
> 2) if there are any --deqp-case options passed into the Test derived
>classes this will cause the tests to fail (conflicting options)
>
> Both of these are resolved by this patch.
>
> CC: Mark Janes
> Signed-off-by: Dylan Baker
> ---
> framework/test/deqp.py | 11 ---
> tests/deqp_gles2.py| 12 +++-
> tests/deqp_gles3.py| 11 +++
> tests/deqp_gles31.py | 12 +++-
> 4 files changed, 29 insertions(+), 17 deletions(-)
>
> diff --git a/framework/test/deqp.py b/framework/test/deqp.py
> index 843cde8..1462ca2 100644
> --- a/framework/test/deqp.py
> +++ b/framework/test/deqp.py
> @@ -61,8 +61,12 @@ def get_option(env_varname, config_option, default=None):
> return opt or default
>
>
> -def gen_caselist_txt(bin_, caselist):
> -"""Generate a caselist.txt and return its path."""
> +def gen_caselist_txt(bin_, caselist, extra_args):
> +"""Generate a caselist.txt and return its path.
> +
> +Extra args should be a list of extra arguments to pass to deqp.
> +
> +"""
> # dEQP is stupid (2014-12-07):
> # 1. To generate the caselist file, dEQP requires that the process's
> # current directory must be that same as that of the executable.
> @@ -76,9 +80,10 @@ def gen_caselist_txt(bin_, caselist):
> # we must *dynamically* generate it during the testrun.
> basedir = os.path.dirname(bin_)
> caselist_path = os.path.join(basedir, caselist)
> +
> # TODO: need to catch some exceptions here...
> subprocess.check_call(
> -[bin_, '--deqp-runmode=txt-caselist'], cwd=basedir)
> +[bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir)
> assert os.path.exists(caselist_path)
> return caselist_path
>
> diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
> index b97cb5d..c4bacb2 100644
> --- a/tests/deqp_gles2.py
> +++ b/tests/deqp_gles2.py
> @@ -24,20 +24,22 @@ from framework.test import deqp
>
> __all__ = ['profile']
>
> -
> # Path to the deqp-gles2 executable.
> _DEQP_GLES2_BIN = deqp.get_option('PIGLIT_DEQP_GLES2_BIN',
>('deqp-gles2', 'bin'))
>
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
> + ('deqp-gles2', 'extra_args'),
> + default='').split()
> +
>
> class DEQPGLES2Test(deqp.DEQPBaseTest):
> deqp_bin = _DEQP_GLES2_BIN
> -extra_args = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
> - ('deqp-gles2', 'extra_args'),
> - default='').split()
> +extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>
>
> profile = deqp.make_profile( # pylint: disable=invalid-name
> deqp.iter_deqp_test_cases(
> -deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt')),
> +deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
> + _EXTRA_ARGS)),
> DEQPGLES2Test)
> diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
> index dfb82c9..d353899 100644
> --- a/tests/deqp_gles3.py
> +++ b/tests/deqp_gles3.py
> @@ -35,6 +35,10 @@ _DEQP_GLES3_EXE = deqp.get_option('PIGLIT_DEQP_GLES3_EXE',
> _DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_MUSTPASS',
> ('deqp-gles3', 'mustpasslist'))
>
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
> + ('deqp-gles3', 'extra_args'),
> + default='').split()
> +
>
> def _get_test_case(root, root_group, outputfile):
> """Parser the test case list of Google Android CTS,
> @@ -73,9 +77,7 @@ def filter_mustpass(caselist_path):
>
> class DEQPGLES3Test(deqp.DEQPBaseTest):
> deqp_bin = _DEQP_GLES3_EXE
> -extra_args = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
> - ('deqp-gles3', 'extra_args'),
> - default='').split()
> +extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>
>
> def __init__(self, *args, **kwargs):
> @@ -84,5 +86,6 @@ class DEQPGLES3Test(deqp.DEQPBaseTest):
>
> profile = deqp.make_profile( # pylint: disable=invalid-name
> deqp.iter_deqp_test_cases(filter_mustpass(
> -deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt'))),
> +deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
> + _EXTRA_ARGS))),
> DEQPGLES3Test)
> diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
> index 423c1bf..b98d2be 100644
> --- a/tests/deqp_gles31.py
> +++ b/tests/deqp_gles31
[Piglit] [PATCH] framework/test/deqp: handle deqp extra args properly in more cases
Currently we blindly pass deqp extra args to each test case, and only to
the test case.
There are two cases where this is problematic:
1) if the extra args need to be passed to the deqp command that
generates the test list (it isn't before this patch)
2) if there are any --deqp-case options passed into the Test derived
classes this will cause the tests to fail (conflicting options)
Both of these are resolved by this patch.
CC: Mark Janes
Signed-off-by: Dylan Baker
---
framework/test/deqp.py | 11 ---
tests/deqp_gles2.py| 12 +++-
tests/deqp_gles3.py| 11 +++
tests/deqp_gles31.py | 12 +++-
4 files changed, 29 insertions(+), 17 deletions(-)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 843cde8..1462ca2 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -61,8 +61,12 @@ def get_option(env_varname, config_option, default=None):
return opt or default
-def gen_caselist_txt(bin_, caselist):
-"""Generate a caselist.txt and return its path."""
+def gen_caselist_txt(bin_, caselist, extra_args):
+"""Generate a caselist.txt and return its path.
+
+Extra args should be a list of extra arguments to pass to deqp.
+
+"""
# dEQP is stupid (2014-12-07):
# 1. To generate the caselist file, dEQP requires that the process's
# current directory must be that same as that of the executable.
@@ -76,9 +80,10 @@ def gen_caselist_txt(bin_, caselist):
# we must *dynamically* generate it during the testrun.
basedir = os.path.dirname(bin_)
caselist_path = os.path.join(basedir, caselist)
+
# TODO: need to catch some exceptions here...
subprocess.check_call(
-[bin_, '--deqp-runmode=txt-caselist'], cwd=basedir)
+[bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir)
assert os.path.exists(caselist_path)
return caselist_path
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index b97cb5d..c4bacb2 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -24,20 +24,22 @@ from framework.test import deqp
__all__ = ['profile']
-
# Path to the deqp-gles2 executable.
_DEQP_GLES2_BIN = deqp.get_option('PIGLIT_DEQP_GLES2_BIN',
('deqp-gles2', 'bin'))
+_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
+ ('deqp-gles2', 'extra_args'),
+ default='').split()
+
class DEQPGLES2Test(deqp.DEQPBaseTest):
deqp_bin = _DEQP_GLES2_BIN
-extra_args = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
- ('deqp-gles2', 'extra_args'),
- default='').split()
+extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(
-deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt')),
+deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
+ _EXTRA_ARGS)),
DEQPGLES2Test)
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index dfb82c9..d353899 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -35,6 +35,10 @@ _DEQP_GLES3_EXE = deqp.get_option('PIGLIT_DEQP_GLES3_EXE',
_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_MUSTPASS',
('deqp-gles3', 'mustpasslist'))
+_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
+ ('deqp-gles3', 'extra_args'),
+ default='').split()
+
def _get_test_case(root, root_group, outputfile):
"""Parser the test case list of Google Android CTS,
@@ -73,9 +77,7 @@ def filter_mustpass(caselist_path):
class DEQPGLES3Test(deqp.DEQPBaseTest):
deqp_bin = _DEQP_GLES3_EXE
-extra_args = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
- ('deqp-gles3', 'extra_args'),
- default='').split()
+extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
def __init__(self, *args, **kwargs):
@@ -84,5 +86,6 @@ class DEQPGLES3Test(deqp.DEQPBaseTest):
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(filter_mustpass(
-deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt'))),
+deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
+ _EXTRA_ARGS))),
DEQPGLES3Test)
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index 423c1bf..b98d2be 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -24,20 +24,22 @@ from framework.test import deqp
__all__ = ['profile']
-
# Path to the deqp-gles3 executable.
_DEQP_GLES31_BIN = deqp.get_option('PIGLIT_DEQP_GLES31_BIN',
('deqp-gles31', 'bin'))
+_EXTRA_ARGS = deqp.get_option('PIGLIT_
