Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
On Thu, 2016-06-02 at 13:24 -0700, Mark Janes wrote: > > Other than the one nit this looks good. Please be sure to let Mark > Janes > > (cc'd) or myself know when you push this, as we'll need to update > our CI > > since this will change some profile names. > > Whoops! This was pushed on May 23, at which point our CI stopped > running the CTS. And now we have a handful of regressions to bisect. Ouch! I totally forgot to ping Mark. I'm very very sorry for the inconveniences. J.A. ___ Piglit mailing list Piglit@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
Dylan Baker writes:
> Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
>> This adds support for running Khronos' deqp-based conformance suite for
>> OpenGL with piglit.
>> ---
>> piglit.conf.example | 10 ++
>> tests/{cts.py => cts_gl45.py} | 43
>> +
>> tests/{cts.py => cts_gles31.py} | 2 +-
>
> Since these run all of the tests I think I'd prefer cts_gles and cts_gl,
> but that's a bit of a nit-pick.
>
>> 3 files changed, 37 insertions(+), 18 deletions(-)
>> copy tests/{cts.py => cts_gl45.py} (56%)
>> rename tests/{cts.py => cts_gles31.py} (97%)
>>
>> diff --git a/piglit.conf.example b/piglit.conf.example
>> index 944d5d9..3cce52a 100644
>> --- a/piglit.conf.example
>> +++ b/piglit.conf.example
>> @@ -95,6 +95,16 @@ testB
>> ; overrides the value set here.
>> ;extra_args=--deqp-visibility hidden
>>
>> +[cts_gl]
>> +; path to the cts opengl executable
>> +; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
>> +;bin=/home/knuth/cts/cts/glcts
>> +
>> +; Space-separated list of extra command line arguments for cts. The
>> +; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
>> +; overrides the value set here.
>> +;extra_args=--deqp-visibility hidden
>> +
>> ; Section for specific oclconform test. One of these sections is required
>> for
>> ; each test list in the oclconform section and must be called:
>> ; oclconform-$testname
>> diff --git a/tests/cts.py b/tests/cts_gl45.py
>> similarity index 56%
>> copy from tests/cts.py
>> copy to tests/cts_gl45.py
>> index 0e64e1b..607b79b 100644
>> --- a/tests/cts.py
>> +++ b/tests/cts_gl45.py
>> @@ -20,25 +20,24 @@
>>
>> """Piglit integration for Khronos CTS tests.
>>
>> -By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
>> -desiring to run only a subset of them should consider using the -t or -x
>> -options to include or exclude tests.
>> +By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43,
>> GL44
>> +and GL45 test cases. Those desiring to run only a subset of them should
>> consider
>> +using the -t or -x options to include or exclude tests.
>>
>> For example:
>> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
>> -exclude ES31 tests)
>> +./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
>>
>> This integration requires some configuration in piglit.conf, or the use of
>> environment variables.
>>
>> In piglit.conf one should set the following:
>> -[cts]:bin -- Path to the glcts binary
>> -[cts]:extra_args -- any extra arguments to be passed to cts (optional)
>> +[cts_gl]:bin -- Path to the glcts binary
>> +[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
>
>>
>> Alternatively (or in addition, since environment variables have precedence),
>> one could set:
>> -PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
>> -PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
>> +PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
>> +PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
>>
>> """
>>
>> @@ -51,9 +50,9 @@ from framework.test import deqp
>>
>> __all__ = ['profile']
>>
>> -_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
>> +_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
>>
>> -_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts',
>> 'extra_args'),
>> +_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl',
>> 'extra_args'),
>>default='').split()
>>
>>
>> @@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
>> return super(DEQPCTSTest, self).extra_args + \
>> [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>>
>> -
>> # Add all of the suites by default, users can use filters to remove them.
>> profile = deqp.make_profile( # pylint: disable=invalid-name
>> itertools.chain(
>> deqp.iter_deqp_test_cases(
>> -deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt',
>> _EXTRA_ARGS)),
>> +deqp.iter_deqp_test_cases(
>> +deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt',
>>
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
Quoting Emil Velikov (2016-05-19 02:52:35) > On 19 May 2016 at 07:33, Juan A. Suarez Romero wrote: > > On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote: > >> I'm 50/50 on it. While I think it probably is nicer to change them, > >> it > >> also creates more churn for people using the functionality. > > > > > > Then, I'll keep as it is. We can always change it in future, if > > required. > > > Wouldn't it be better to get all the churn in one go, considering one > already needs to tweak things, manually (the cts.py > cts_gles.py > rename) ? > > -Emil Agreed. Either we should change the name now or never do it. I think I prefer doing it now. Dylan signature.asc Description: signature ___ Piglit mailing list Piglit@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
On 19 May 2016 at 07:33, Juan A. Suarez Romero wrote: > On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote: >> I'm 50/50 on it. While I think it probably is nicer to change them, >> it >> also creates more churn for people using the functionality. > > > Then, I'll keep as it is. We can always change it in future, if > required. > Wouldn't it be better to get all the churn in one go, considering one already needs to tweak things, manually (the cts.py > cts_gles.py rename) ? -Emil ___ Piglit mailing list Piglit@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote: > I'm 50/50 on it. While I think it probably is nicer to change them, > it > also creates more churn for people using the functionality. Then, I'll keep as it is. We can always change it in future, if required. Thanks a lot! J.A. ___ Piglit mailing list Piglit@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
Quoting Juan A. Suarez Romero (2016-05-18 09:27:34)
> On Wed, 2016-05-18 at 09:20 -0700, Dylan Baker wrote:
> > Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> > > This adds support for running Khronos' deqp-based conformance suite
> > for
> > > OpenGL with piglit.
> > > ---
> > > piglit.conf.example | 10 ++
> > > tests/{cts.py => cts_gl45.py} | 43 +--
> > --
> > > tests/{cts.py => cts_gles31.py} | 2 +-
> >
> > Since these run all of the tests I think I'd prefer cts_gles and
> > cts_gl,
> > but that's a bit of a nit-pick.
>
> What about the envvars / piglit.conf?
>
> I keep PIGLIT_CTS_BIN for gles, while using PIGLIT_CTS_GL_BIN for gl.
>
> Does it make sense to rename the envvar and use also
> PIGLIT_CTS_GLES_BIN / PIGLIT_CTS_GLES_EXTRA_ARGS for gles, and
> PIGLIT_CTS_GL_BIN / PIGLIT_CTS_GL_EXTRA_ARGS for gl?
>
>
> J.A.
I'm 50/50 on it. While I think it probably is nicer to change them, it
also creates more churn for people using the functionality.
My r-b remains whichever way you decide to go.
Dylan
signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
On Wed, 2016-05-18 at 09:20 -0700, Dylan Baker wrote:
> Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> > This adds support for running Khronos' deqp-based conformance suite
> for
> > OpenGL with piglit.
> > ---
> > piglit.conf.example | 10 ++
> > tests/{cts.py => cts_gl45.py} | 43 +--
> --
> > tests/{cts.py => cts_gles31.py} | 2 +-
>
> Since these run all of the tests I think I'd prefer cts_gles and
> cts_gl,
> but that's a bit of a nit-pick.
What about the envvars / piglit.conf?
I keep PIGLIT_CTS_BIN for gles, while using PIGLIT_CTS_GL_BIN for gl.
Does it make sense to rename the envvar and use also
PIGLIT_CTS_GLES_BIN / PIGLIT_CTS_GLES_EXTRA_ARGS for gles, and
PIGLIT_CTS_GL_BIN / PIGLIT_CTS_GL_EXTRA_ARGS for gl?
J.A.
signature.asc
Description: This is a digitally signed message part
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> This adds support for running Khronos' deqp-based conformance suite for
> OpenGL with piglit.
> ---
> piglit.conf.example | 10 ++
> tests/{cts.py => cts_gl45.py} | 43
> +
> tests/{cts.py => cts_gles31.py} | 2 +-
Since these run all of the tests I think I'd prefer cts_gles and cts_gl,
but that's a bit of a nit-pick.
> 3 files changed, 37 insertions(+), 18 deletions(-)
> copy tests/{cts.py => cts_gl45.py} (56%)
> rename tests/{cts.py => cts_gles31.py} (97%)
>
> diff --git a/piglit.conf.example b/piglit.conf.example
> index 944d5d9..3cce52a 100644
> --- a/piglit.conf.example
> +++ b/piglit.conf.example
> @@ -95,6 +95,16 @@ testB
> ; overrides the value set here.
> ;extra_args=--deqp-visibility hidden
>
> +[cts_gl]
> +; path to the cts opengl executable
> +; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
> +;bin=/home/knuth/cts/cts/glcts
> +
> +; Space-separated list of extra command line arguments for cts. The
> +; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
> +; overrides the value set here.
> +;extra_args=--deqp-visibility hidden
> +
> ; Section for specific oclconform test. One of these sections is required
> for
> ; each test list in the oclconform section and must be called:
> ; oclconform-$testname
> diff --git a/tests/cts.py b/tests/cts_gl45.py
> similarity index 56%
> copy from tests/cts.py
> copy to tests/cts_gl45.py
> index 0e64e1b..607b79b 100644
> --- a/tests/cts.py
> +++ b/tests/cts_gl45.py
> @@ -20,25 +20,24 @@
>
> """Piglit integration for Khronos CTS tests.
>
> -By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
> -desiring to run only a subset of them should consider using the -t or -x
> -options to include or exclude tests.
> +By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
> +and GL45 test cases. Those desiring to run only a subset of them should
> consider
> +using the -t or -x options to include or exclude tests.
>
> For example:
> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
> -exclude ES31 tests)
> +./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
>
> This integration requires some configuration in piglit.conf, or the use of
> environment variables.
>
> In piglit.conf one should set the following:
> -[cts]:bin -- Path to the glcts binary
> -[cts]:extra_args -- any extra arguments to be passed to cts (optional)
> +[cts_gl]:bin -- Path to the glcts binary
> +[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
>
> Alternatively (or in addition, since environment variables have precedence),
> one could set:
> -PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
> -PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
> +PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
> +PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
>
> """
>
> @@ -51,9 +50,9 @@ from framework.test import deqp
>
> __all__ = ['profile']
>
> -_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
> +_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
>
> -_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl',
> 'extra_args'),
>default='').split()
>
>
> @@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
> return super(DEQPCTSTest, self).extra_args + \
> [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>
> -
> # Add all of the suites by default, users can use filters to remove them.
> profile = deqp.make_profile( # pylint: disable=invalid-name
> itertools.chain(
> deqp.iter_deqp_test_cases(
> -deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt',
> _EXTRA_ARGS)),
> deqp.iter_deqp_test_cases(
> -deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt',
> _EXTRA_ARGS)),
> +deq
[Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
This adds support for running Khronos' deqp-based conformance suite for
OpenGL with piglit.
---
piglit.conf.example | 10 ++
tests/{cts.py => cts_gl45.py} | 43 +
tests/{cts.py => cts_gles31.py} | 2 +-
3 files changed, 37 insertions(+), 18 deletions(-)
copy tests/{cts.py => cts_gl45.py} (56%)
rename tests/{cts.py => cts_gles31.py} (97%)
diff --git a/piglit.conf.example b/piglit.conf.example
index 944d5d9..3cce52a 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -95,6 +95,16 @@ testB
; overrides the value set here.
;extra_args=--deqp-visibility hidden
+[cts_gl]
+; path to the cts opengl executable
+; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
+;bin=/home/knuth/cts/cts/glcts
+
+; Space-separated list of extra command line arguments for cts. The
+; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
+; overrides the value set here.
+;extra_args=--deqp-visibility hidden
+
; Section for specific oclconform test. One of these sections is required for
; each test list in the oclconform section and must be called:
; oclconform-$testname
diff --git a/tests/cts.py b/tests/cts_gl45.py
similarity index 56%
copy from tests/cts.py
copy to tests/cts_gl45.py
index 0e64e1b..607b79b 100644
--- a/tests/cts.py
+++ b/tests/cts_gl45.py
@@ -20,25 +20,24 @@
"""Piglit integration for Khronos CTS tests.
-By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
-desiring to run only a subset of them should consider using the -t or -x
-options to include or exclude tests.
+By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
+and GL45 test cases. Those desiring to run only a subset of them should
consider
+using the -t or -x options to include or exclude tests.
For example:
-./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
-exclude ES31 tests)
+./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
This integration requires some configuration in piglit.conf, or the use of
environment variables.
In piglit.conf one should set the following:
-[cts]:bin -- Path to the glcts binary
-[cts]:extra_args -- any extra arguments to be passed to cts (optional)
+[cts_gl]:bin -- Path to the glcts binary
+[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
Alternatively (or in addition, since environment variables have precedence),
one could set:
-PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
-PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
+PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
+PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
"""
@@ -51,9 +50,9 @@ from framework.test import deqp
__all__ = ['profile']
-_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
+_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
-_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
+_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl',
'extra_args'),
default='').split()
@@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
return super(DEQPCTSTest, self).extra_args + \
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
-
# Add all of the suites by default, users can use filters to remove them.
profile = deqp.make_profile( # pylint: disable=invalid-name
itertools.chain(
deqp.iter_deqp_test_cases(
-deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
+deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.iter_deqp_test_cases(
+deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt',
_EXTRA_ARGS)),
deqp.iter_deqp_test_cases(
-deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
+deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt',
_EXTRA_ARGS)),
deqp.iter_deqp_test_cases(
-deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt',
_EXTRA_ARGS)),
+deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt',
_EXTRA_ARGS)),
deqp.iter_deqp_test_cases(
-deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
- _E
