This adds the option of using a python module, rather than calling out to the shell for xz support.
v4: - add this patch Signed-off-by: Dylan Baker <dylanx.c.ba...@intel.com> --- README | 3 + framework/backends/compression.py | 129 ++++++++++++++++++++------------------ 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/README b/README index 1968221..773a350 100644 --- a/README +++ b/README @@ -45,6 +45,9 @@ Optionally, you can install the following: - lxml. An accelerated python xml library using libxml2 (http://lxml.de/) - simplejson. A fast C based implementation of the python json library. (https://simplejson.readthedocs.org/en/latest/) + - backports.lzma. A packport of python3's lzma module to python2, + this enables fast native xz (de)compression in piglit for results files + (https://github.com/peterjc/backports.lzma) Now configure the build system: diff --git a/framework/backends/compression.py b/framework/backends/compression.py index ef97c95..4bb4b0f 100644 --- a/framework/backends/compression.py +++ b/framework/backends/compression.py @@ -73,80 +73,87 @@ DECOMPRESSORS = { # TODO: in python3 there is builtin xz support, and doesn't need this madness # If there is an xz binary then try calling out to that try: - with open(os.devnull, 'w') as d: - subprocess.check_call(['xz'], stderr=d) -except subprocess.CalledProcessError as e: - if e.returncode == 1: - import contextlib - try: - import cStringIO as StringIO - except ImportError: - import StringIO + import backports.lzma + + COMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='w') + DECOMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='r') + COMPRESSION_SUFFIXES += ['.xz'] +except ImportError: + try: + with open(os.devnull, 'w') as d: + subprocess.check_call(['xz'], stderr=d) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + import contextlib + try: + import cStringIO as StringIO + except ImportError: + import StringIO - @contextlib.contextmanager - def _compress_xz(filename): - """Emulates an open function in write mode for xz. + @contextlib.contextmanager + def _compress_xz(filename): + """Emulates an open function in write mode for xz. - Python 2.x doesn't support xz, but it's dang useful. This function calls - out to the shell and tries to use xz from the environment to get xz - compression. + Python 2.x doesn't support xz, but it's dang useful. This function calls + out to the shell and tries to use xz from the environment to get xz + compression. - This obviously won't work without a working xz binary. + This obviously won't work without a working xz binary. - This function tries to emulate the default values of the lzma module in - python3 as much as possible + This function tries to emulate the default values of the lzma module in + python3 as much as possible - """ - if filename.endswith('.xz'): - filename = filename[:-2] + """ + if filename.endswith('.xz'): + filename = filename[:-2] - with open(filename, 'w') as f: - yield f + with open(filename, 'w') as f: + yield f - try: - subprocess.check_call(['xz', '--compress', '-9', filename]) - except OSError as e: - if e.errno == errno.ENOENT: - raise exceptions.PiglitFatalError('No xz binary available') - raise + try: + subprocess.check_call(['xz', '--compress', '-9', filename]) + except OSError as e: + if e.errno == errno.ENOENT: + raise exceptions.PiglitFatalError('No xz binary available') + raise - @contextlib.contextmanager - def _decompress_xz(filename): - """Eumlates an option function in read mode for xz. + @contextlib.contextmanager + def _decompress_xz(filename): + """Eumlates an option function in read mode for xz. - See the comment in _compress_xz for more information. + See the comment in _compress_xz for more information. - This module tries to emulate the lzma module as much as possible + This module tries to emulate the lzma module as much as possible - """ - if not filename.endswith('.xz'): - filename = '{}.xz'.format(filename) + """ + if not filename.endswith('.xz'): + filename = '{}.xz'.format(filename) - print('decompress: ' + filename) + print('decompress: ' + filename) - try: - string = subprocess.check_output( - ['xz', '--decompress', '--stdout', filename]) - except OSError as e: - if e.errno == errno.ENOENT: - raise exceptions.PiglitFatalError('No xz binary available') - raise - - # We need a file-like object, so the contents must be placed in a StringIO - # object. - io = StringIO.StringIO() - io.write(string) - io.seek(0) - - yield io - - io.close() - - COMPRESSORS['xz'] = _compress_xz - DECOMPRESSORS['xz'] = _decompress_xz - COMPRESSION_SUFFIXES += ['.xz'] -except OSError: - pass + try: + string = subprocess.check_output( + ['xz', '--decompress', '--stdout', filename]) + except OSError as e: + if e.errno == errno.ENOENT: + raise exceptions.PiglitFatalError('No xz binary available') + raise + + # We need a file-like object, so the contents must be placed in a StringIO + # object. + io = StringIO.StringIO() + io.write(string) + io.seek(0) + + yield io + + io.close() + + COMPRESSORS['xz'] = _compress_xz + DECOMPRESSORS['xz'] = _decompress_xz + COMPRESSION_SUFFIXES += ['.xz'] + except OSError: + pass def _set_mode(): -- 2.4.5 _______________________________________________ Piglit mailing list Piglit@lists.freedesktop.org http://lists.freedesktop.org/mailman/listinfo/piglit