This adds support to compress results with gzip compression. This
reduces the size of json results significantly (from 21M to 1.6M when
running the quick profile (which is about 7% of the uncompressed size).
v4: - add additional tests
Signed-off-by: Dylan Baker <dylanx.c.ba...@intel.com>
---
framework/backends/compression.py | 9 ++-
framework/tests/compressed_backend_tests.py | 89 ++++++++++++++++++++++++++++-
piglit.conf.example | 6 +-
3 files changed, 97 insertions(+), 7 deletions(-)
diff --git a/framework/backends/compression.py
b/framework/backends/compression.py
index 37383e4..44b8b9e 100644
--- a/framework/backends/compression.py
+++ b/framework/backends/compression.py
@@ -43,21 +43,24 @@ they're passing unicode and not bytes.
from __future__ import print_function, absolute_import, division
import functools
+import gzip
import os
from framework import exceptions
from framework.core import PIGLIT_CONFIG
-COMPRESSION_SUFFIXES = []
+COMPRESSION_SUFFIXES = ['.gz']
-DEFAULT = 'none'
+DEFAULT = 'gz'
COMPRESSORS = {
+ 'gz': functools.partial(gzip.open, mode='w'),
'none': functools.partial(open, mode='w'),
}
DECOMPRESSORS = {
- 'none': functools.partial(open, mode='r')
+ 'gz': functools.partial(gzip.open, mode='r'),
+ 'none': functools.partial(open, mode='r'),
}
diff --git a/framework/tests/compressed_backend_tests.py
b/framework/tests/compressed_backend_tests.py
index eb40f0a..897a154 100644
--- a/framework/tests/compressed_backend_tests.py
+++ b/framework/tests/compressed_backend_tests.py
@@ -32,10 +32,31 @@ import functools
import nose.tools as nt
from framework.tests import utils
-from framework.backends import compression
+from framework.backends import compression, abstract
# pylint: disable=line-too-long,protected-access
+# Helpers
+
+
+class _TestBackend(abstract.FileBackend):
+ """A class for testing backend compression."""
+ _file_extension = 'test_extension'
+
+ def initialize(self, *args, **kwargs): # pylint: disable=unused-argument
+ os.mkdir(os.path.join(self._dest, 'tests'))
+
+ def finalize(self, *args, **kwargs): # pylint: disable=unused-argument
+ tests = os.path.join(self._dest, 'tests')
+ with self._write_final(os.path.join(self._dest, 'results.txt')) as f:
+ for file_ in os.listdir(tests):
+ with open(os.path.join(tests, file_), 'r') as t:
+ f.write(t.read())
+
+ @staticmethod
+ def _write(f, name, data): # pylint: disable=arguments-differ
+ f.write('{}: {}'.format(name, data))
+
def _add_compression(value):
"""Decorator that temporarily adds support for a compression method."""
@@ -60,6 +81,31 @@ def _add_compression(value):
return _wrapper
+def _set_compression_mode(mode):
+ """Change the compression mode for one test."""
+
+ def _wrapper(func):
+ """The actual decorator."""
+
+ @functools.wraps(func)
+ @utils.set_env(PIGLIT_COMPRESSION=mode)
+ def _inner(*args, **kwargs):
+ """The called function."""
+ restore = compression.MODE
+ compression.MODE = compression._set_mode()
+ compression.COMPRESSOR = compression.COMPRESSORS[compression.MODE]
+
+ try:
+ func(*args, **kwargs)
+ finally:
+ compression.MODE = restore
+ compression.COMPRESSOR =
compression.COMPRESSORS[compression.MODE]
+
+ return _inner
+
+ return _wrapper
+
+
def _test_compressor(mode):
"""Helper to simplify testing compressors."""
func = compression.COMPRESSORS[mode]
@@ -83,6 +129,29 @@ def _test_decompressor(mode):
nt.eq_(f.read(), 'foo')
+def _test_extension():
+ """Create an final file and return the extension."""
+ with utils.tempdir() as d:
+ obj = _TestBackend(d)
+ obj.initialize()
+ with obj.write_test('foo') as t:
+ t({'result': 'foo'})
+
+ obj.finalize()
+
+ for each in os.listdir(d):
+ if each.startswith('results.txt'):
+ ext = os.path.splitext(each)[1]
+ break
+ else:
+ raise utils.TestFailure('No results file generated')
+
+ return ext
+
+
+# Tests
+
+
@utils.no_error
def test_compress_none():
"""framework.backends.compression: can compress to 'none'"""
@@ -94,6 +163,7 @@ def test_decompress_none():
_test_decompressor('none')
+
@_add_compression('foobar')
@utils.set_env(PIGLIT_COMPRESSION='foobar')
def test_set_mode_env():
@@ -114,3 +184,20 @@ def test_set_mode_piglit_conf():
def test_set_mode_default():
"""framework.backends.compression._set_mode: uses DEFAULT if env and
piglit.conf are unset"""
nt.eq_(compression._set_mode(), compression.DEFAULT)
+
+
+@utils.no_error
+def test_compress_gz():
+ """framework.backends.compression: can compress to 'gz'"""
+ _test_compressor('gz')
+
+
+def test_decompress_gz():
+ """framework.backends.compression: can decompress from 'gz'"""
+ _test_decompressor('gz')
+
+
+@_set_compression_mode('gz')
+def test_gz_output():
+ """framework.backends: when using gz compression a gz file is created"""
+ nt.eq_(_test_extension(), '.gz')
diff --git a/piglit.conf.example b/piglit.conf.example
index 49bc8b6..baf8108 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -111,9 +111,9 @@ run_test=./%(test_name)s
;backend=json
; Set the default compression method to use,
-; May be one of: 'none'
-; Default: none (Note that this may change in the future)
-;compression=none
+; May be one of: 'none', 'gz'
+; Default: 'gz'
+;compression=gz
[expected-failures]
; Provide a list of test names that are expected to fail. These tests
--
2.4.5
_______________________________________________
Piglit mailing list
Piglit@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/piglit
