Author: Ronan Lamy <ronan.l...@gmail.com> Branch: py3k Changeset: r87007:fbcb88bba801 Date: 2016-09-11 20:26 +0100 http://bitbucket.org/pypy/pypy/changeset/fbcb88bba801/
Log: hg merge default diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst --- a/pypy/doc/release-pypy2.7-v5.4.1.rst +++ b/pypy/doc/release-pypy2.7-v5.4.1.rst @@ -9,16 +9,16 @@ this was unfortunately left out of 5.4.0. My apologies to the new contributors - * Allow tests run with `-A` to find `libm.so` even if it is a script not a + * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a dynamically loadable file - * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython + * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython - * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors + * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors - * Fix for an issue where os.access() accepted a float for mode + * Fix for an issue for translating the sandbox - * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up + * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up the last byte of a unicode string sometimes * Update built-in cffi_ to version 1.8.1 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -12,4 +12,7 @@ Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews in numpypy - +.. branch: force-virtual-state +Improve merging of virtual states in the JIT in order to avoid jumping to the +preamble. Accomplished by allocating virtual objects where non-virtuals are +expected. diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -18,6 +18,8 @@ from .support import c_compile +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" + @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): 1/0 @@ -53,7 +55,48 @@ libraries=libraries) return soname -def compile_extension_module(space, modname, include_dirs=[], +class SystemCompilationInfo(object): + """Bundles all the generic information required to compile extensions. + + Note: here, 'system' means OS + target interpreter + test config + ... + """ + def __init__(self, include_extra=None, compile_extra=None, link_extra=None, + extra_libs=None, ext=None): + self.include_extra = include_extra or [] + self.compile_extra = compile_extra + self.link_extra = link_extra + self.extra_libs = extra_libs + self.ext = ext + +def get_cpyext_info(space): + from pypy.module.imp.importing import get_so_extension + state = space.fromcache(State) + api_library = state.api_lib + if sys.platform == 'win32': + libraries = [api_library] + # '%s' undefined; assuming extern returning int + compile_extra = ["/we4013"] + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] + else: + libraries = [] + if sys.platform.startswith('linux'): + compile_extra = [ + "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"] + link_extra = ["-g"] + else: + compile_extra = link_extra = None + return SystemCompilationInfo( + include_extra=api.include_dirs, + compile_extra=compile_extra, + link_extra=link_extra, + extra_libs=libraries, + ext=get_so_extension(space)) + + +def compile_extension_module(sys_info, modname, include_dirs=[], source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native @@ -65,35 +108,15 @@ Any extra keyword arguments are passed on to ExternalCompilationInfo to build the module (so specify your source with one of those). """ - state = space.fromcache(State) - api_library = state.api_lib - if sys.platform == 'win32': - libraries = [api_library] - # '%s' undefined; assuming extern returning int - compile_extra = ["/we4013"] - # prevent linking with PythonXX.lib - w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] - link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % - (space.int_w(w_maj), space.int_w(w_min))] - else: - libraries = [] - if sys.platform.startswith('linux'): - compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"] - link_extra = ["-g"] - else: - compile_extra = link_extra = None - modname = modname.split('.')[-1] soname = create_so(modname, - include_dirs=api.include_dirs + include_dirs, - source_files=source_files, - source_strings=source_strings, - compile_extra=compile_extra, - link_extra=link_extra, - libraries=libraries) - from pypy.module.imp.importing import get_so_extension - ext = get_so_extension(space) - pydname = soname.new(purebasename=modname, ext=ext) + include_dirs=sys_info.include_extra + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=sys_info.compile_extra, + link_extra=sys_info.link_extra, + libraries=sys_info.extra_libs) + pydname = soname.new(purebasename=modname, ext=sys_info.ext) soname.rename(pydname) return str(pydname) @@ -106,18 +129,8 @@ raise RuntimeError("This interpreter does not define a filename " "suffix for C extensions!") -def compile_extension_module_applevel(space, modname, include_dirs=[], - source_files=None, source_strings=None): - """ - Build an extension module and return the filename of the resulting native - code file. - - modname is the name of the module, possibly including dots if it is a module - inside a package. - - Any extra keyword arguments are passed on to ExternalCompilationInfo to - build the module (so specify your source with one of those). - """ +def get_sys_info_app(): + from distutils.sysconfig import get_python_inc if sys.platform == 'win32': compile_extra = ["/we4013"] link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] @@ -128,18 +141,13 @@ compile_extra = [ "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] link_extra = None + ext = get_so_suffix() + return SystemCompilationInfo( + include_extra=[get_python_inc()], + compile_extra=compile_extra, + link_extra=link_extra, + ext=get_so_suffix()) - modname = modname.split('.')[-1] - soname = create_so(modname, - include_dirs=[space.include_dir] + include_dirs, - source_files=source_files, - source_strings=source_strings, - compile_extra=compile_extra, - link_extra=link_extra) - ext = get_so_suffix() - pydname = soname.new(purebasename=modname, ext=ext) - soname.rename(pydname) - return str(pydname) def freeze_refcnts(self): rawrefcount._dont_free_any_more() @@ -154,9 +162,7 @@ class FakeSpace(object): """Like TinyObjSpace, but different""" def __init__(self, config): - from distutils.sysconfig import get_python_inc self.config = config - self.include_dir = get_python_inc() def passthrough(self, arg): return arg @@ -273,11 +279,11 @@ "the test actually passed in the first place; if it failed " "it is likely to reach this place.") - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_only_import(self): import cpyext - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_load_error(self): import cpyext raises(ImportError, cpyext.load_module, "missing.file", "foo") @@ -305,26 +311,23 @@ def setup_method(self, func): @gateway.unwrap_spec(name=str) def compile_module(space, name, - w_separate_module_files=None, - w_separate_module_sources=None): + w_source_files=None, + w_source_strings=None): """ Build an extension module linked against the cpyext api library. """ - if not space.is_none(w_separate_module_files): - separate_module_files = space.unwrap(w_separate_module_files) - assert separate_module_files is not None + if not space.is_none(w_source_files): + source_files = space.unwrap(w_source_files) else: - separate_module_files = [] - if not space.is_none(w_separate_module_sources): - separate_module_sources = space.listview_bytes( - w_separate_module_sources) - assert separate_module_sources is not None + source_files = None + if not space.is_none(w_source_strings): + source_strings = space.listview_bytes(w_source_strings) else: - separate_module_sources = [] - pydname = self.compile_extension_module( - space, name, - source_files=separate_module_files, - source_strings=separate_module_sources) + source_strings = None + pydname = compile_extension_module( + self.sys_info, name, + source_files=source_files, + source_strings=source_strings) return space.wrap(pydname) @gateway.unwrap_spec(name=str, init='str_or_None', body=str, @@ -375,7 +378,7 @@ filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") kwds = dict(source_files=[filename]) - mod = self.compile_extension_module(space, name, + mod = compile_extension_module(self.sys_info, name, include_dirs=include_dirs, **kwds) if load_it: @@ -475,11 +478,11 @@ return run def wrap(func): return func - self.compile_extension_module = compile_extension_module_applevel + self.sys_info = get_sys_info_app() else: interp2app = gateway.interp2app wrap = self.space.wrap - self.compile_extension_module = compile_extension_module + self.sys_info = get_cpyext_info(self.space) self.w_compile_module = wrap(interp2app(compile_module)) self.w_import_module = wrap(interp2app(import_module)) self.w_reimport_module = wrap(interp2app(reimport_module)) @@ -629,10 +632,10 @@ skip('record_imported_module not supported in runappdirect mode') # Build the extensions. banana = self.compile_module( - "apple.banana", separate_module_files=[self.here + 'banana.c']) + "apple.banana", source_files=[self.here + 'banana.c']) self.record_imported_module("apple.banana") date = self.compile_module( - "cherry.date", separate_module_files=[self.here + 'date.c']) + "cherry.date", source_files=[self.here + 'date.c']) self.record_imported_module("cherry.date") # Set up some package state so that the extensions can actually be @@ -901,7 +904,7 @@ ]) raises(SystemError, mod.newexc, "name", Exception, {}) - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test') + @pytest.mark.skipif(only_pypy, reason='pypy specific test') def test_hash_pointer(self): mod = self.import_extension('foo', [ ('get_hash', 'METH_NOARGS', @@ -952,7 +955,7 @@ print(p) assert 'py' in p - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_get_version(self): mod = self.import_extension('foo', [ ('get_version', 'METH_NOARGS', diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -88,14 +88,10 @@ from _numpypy import multiarray as np module = self.import_module(name='buffer_test') get_buffer_info = module.get_buffer_info - # test_export_flags from numpy test_multiarray raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) - # test_relaxed_strides from numpy test_multiarray - arr = np.zeros((1, 10)) - if arr.flags.f_contiguous: - shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS']) - assert strides[0] == 8 - arr = np.ones((10, 1), order='F') - shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) - assert strides[-1] == 8 - + arr = np.zeros((1, 10), order='F') + shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS']) + assert strides[0] == 8 + arr = np.zeros((10, 1), order='C') + shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) + assert strides[-1] == 8 diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py --- a/pypy/module/cpyext/test/test_thread.py +++ b/pypy/module/cpyext/test/test_thread.py @@ -1,12 +1,13 @@ import sys -import py, pytest +import pytest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class AppTestThread(AppTestCpythonExtensionBase): - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_get_thread_ident(self): module = self.import_extension('foo', [ ("get_thread_ident", "METH_NOARGS", @@ -33,7 +34,7 @@ assert results[0][0] != results[1][0] - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_acquire_lock(self): module = self.import_extension('foo', [ ("test_acquire_lock", "METH_NOARGS", @@ -57,7 +58,7 @@ ]) module.test_acquire_lock() - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_release_lock(self): module = self.import_extension('foo', [ ("test_release_lock", "METH_NOARGS", @@ -79,7 +80,7 @@ ]) module.test_release_lock() - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_tls(self): module = self.import_extension('foo', [ ("create_key", "METH_NOARGS", diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py --- a/pypy/module/cpyext/test/test_version.py +++ b/pypy/module/cpyext/test/test_version.py @@ -3,6 +3,7 @@ import py, pytest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" def test_pragma_version(): from pypy.module.sys.version import CPYTHON_VERSION @@ -41,11 +42,9 @@ assert module.py_minor_version == sys.version_info.minor assert module.py_micro_version == sys.version_info.micro - #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_pypy_versions(self): import sys - if '__pypy__' not in sys.builtin_module_names: - py.test.skip("pypy only test") init = """ if (Py_IsInitialized()) { PyObject *m = Py_InitModule("foo", NULL); diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -95,8 +95,9 @@ ) setuppkg("pkg.pkg2", a='', b='') setuppkg("pkg.withall", - __init__ = "__all__ = ['foobar']", - foobar = "found = 123") + __init__ = "__all__ = ['foobar', 'barbaz']", + foobar = "found = 123", + barbaz = "other = 543") setuppkg("pkg.withoutall", __init__ = "", foobar = "found = 123") @@ -724,6 +725,7 @@ d = {} exec("from pkg.withall import *", d) assert d["foobar"].found == 123 + assert d["barbaz"].other == 543 def test_import_star_does_not_find_submodules_without___all__(self): for case in ["not-imported-yet", "already-imported"]: diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -392,31 +392,39 @@ extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None]) return extobj_w + +_reflected_ops = { + 'add': 'radd', + 'subtract': 'rsub', + 'multiply': 'rmul', + 'divide': 'rdiv', + 'true_divide': 'rtruediv', + 'floor_divide': 'rfloordiv', + 'remainder': 'rmod', + 'power': 'rpow', + 'left_shift': 'rlshift', + 'right_shift': 'rrshift', + 'bitwise_and': 'rand', + 'bitwise_xor': 'rxor', + 'bitwise_or': 'ror', + #/* Comparisons */ + 'equal': 'eq', + 'not_equal': 'ne', + 'greater': 'lt', + 'less': 'gt', + 'greater_equal': 'le', + 'less_equal': 'ge', +} + +for key, value in _reflected_ops.items(): + _reflected_ops[key] = "__" + value + "__" +del key +del value + def _has_reflected_op(space, w_obj, op): - refops ={ 'add': 'radd', - 'subtract': 'rsub', - 'multiply': 'rmul', - 'divide': 'rdiv', - 'true_divide': 'rtruediv', - 'floor_divide': 'rfloordiv', - 'remainder': 'rmod', - 'power': 'rpow', - 'left_shift': 'rlshift', - 'right_shift': 'rrshift', - 'bitwise_and': 'rand', - 'bitwise_xor': 'rxor', - 'bitwise_or': 'ror', - #/* Comparisons */ - 'equal': 'eq', - 'not_equal': 'ne', - 'greater': 'lt', - 'less': 'gt', - 'greater_equal': 'le', - 'less_equal': 'ge', - } - if op not in refops: + if op not in _reflected_ops: return False - return space.getattr(w_obj, space.wrap('__' + refops[op] + '__')) is not None + return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None def safe_casting_mode(casting): assert casting is not None diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -365,6 +365,13 @@ def visitor_dispatch_virtual_type(self, visitor): raise NotImplementedError("abstract") + def make_guards(self, op, short, optimizer): + from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0 + op = ResOperation(rop.INT_EQ, [op, CONST_0]) + short.append(op) + op = ResOperation(rop.GUARD_FALSE, [op]) + short.append(op) + class RawBufferPtrInfo(AbstractRawPtrInfo): buffer = None diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7592,7 +7592,7 @@ ops = """ [i0] p1 = new_with_vtable(descr=nodesize) - cond_call(1, 123, p1, descr=clear_vable) + cond_call(i0, 123, p1, descr=clear_vable) jump(i0) """ expected = """ diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -16,7 +16,7 @@ from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints -class UnrollableOptimizer(Optimizer): +class UnrollableOptimizer(Optimizer): def force_op_from_preamble(self, preamble_op): if isinstance(preamble_op, PreambleOp): if self.optunroll.short_preamble_producer is None: @@ -120,7 +120,8 @@ assert op.get_forwarded() is None if check_newops: assert not self.optimizer._newoperations - + + def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo): info, newops = self.optimizer.propagate_all_forward( trace.get_iter(), call_pure_results, flush=False) @@ -156,7 +157,7 @@ current_vs = self.get_virtual_state(end_jump.getarglist()) # pick the vs we want to jump to assert isinstance(celltoken, JitCellToken) - + target_virtual_state = self.pick_virtual_state(current_vs, state.virtual_state, celltoken.target_tokens) @@ -180,17 +181,27 @@ self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), - self.optimizer._newoperations) + self.optimizer._newoperations) try: - new_virtual_state = self.jump_to_existing_trace(end_jump, label_op, - state.runtime_boxes) + new_virtual_state = self.jump_to_existing_trace( + end_jump, label_op, state.runtime_boxes, force_boxes=False) except InvalidLoop: # inlining short preamble failed, jump to preamble self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) + + if new_virtual_state is not None: + # Attempt to force virtual boxes in order to avoid jumping + # to the preamble. + try: + new_virtual_state = self.jump_to_existing_trace( + end_jump, label_op, state.runtime_boxes, force_boxes=True) + except InvalidLoop: + pass + if new_virtual_state is not None: self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, @@ -199,7 +210,7 @@ self.disable_retracing_if_max_retrace_guards( self.optimizer._newoperations, target_token) - + return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) @@ -241,7 +252,8 @@ for a in jump_op.getarglist(): self.optimizer.force_box_for_end_of_preamble(a) try: - vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes) + vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, + force_boxes=False) except InvalidLoop: return self.jump_to_preamble(cell_token, jump_op, info) if vs is None: @@ -252,6 +264,14 @@ cell_token.retraced_count += 1 debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit)) else: + # Try forcing boxes to avoid jumping to the preamble + try: + vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, + force_boxes=True) + except InvalidLoop: + pass + if vs is None: + return info, self.optimizer._newoperations[:] debug_print("Retrace count reached, jumping to preamble") return self.jump_to_preamble(cell_token, jump_op, info) exported_state = self.export_state(info.jump_op.getarglist(), @@ -288,7 +308,7 @@ return info, self.optimizer._newoperations[:] - def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes): + def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False): jitcelltoken = jump_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) virtual_state = self.get_virtual_state(jump_op.getarglist()) @@ -299,7 +319,8 @@ continue try: extra_guards = target_virtual_state.generate_guards( - virtual_state, args, runtime_boxes, self.optimizer) + virtual_state, args, runtime_boxes, self.optimizer, + force_boxes=force_boxes) patchguardop = self.optimizer.patchguardop for guard in extra_guards.extra_guards: if isinstance(guard, GuardResOp): @@ -308,8 +329,18 @@ self.send_extra_operation(guard) except VirtualStatesCantMatch: continue - args, virtuals = target_virtual_state.make_inputargs_and_virtuals( - args, self.optimizer) + + # When force_boxes == True, creating the virtual args can fail when + # components of the virtual state alias. If this occurs, we must + # recompute the virtual state as boxes will have been forced. + try: + args, virtuals = target_virtual_state.make_inputargs_and_virtuals( + args, self.optimizer, force_boxes=force_boxes) + except VirtualStatesCantMatch: + assert force_boxes + virtual_state = self.get_virtual_state(args) + continue + short_preamble = target_token.short_preamble try: extra = self.inline_short_preamble(args + virtuals, args, @@ -452,7 +483,7 @@ # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetargs, self.optimizer) - + self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, @@ -497,7 +528,7 @@ * runtime_boxes - runtime values for boxes, necessary when generating guards to jump to """ - + def __init__(self, end_args, next_iteration_args, virtual_state, exported_infos, short_boxes, renamed_inputargs, short_inputargs, runtime_boxes, memo): diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -4,7 +4,7 @@ ArrayStructInfo, AbstractStructPtrInfo from rpython.jit.metainterp.optimizeopt.intutils import \ MININT, MAXINT, IntBound, IntLowerBound -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ +from rpython.jit.metainterp.resoperation import rop, ResOperation, \ InputArgInt, InputArgRef, InputArgFloat from rpython.rlib.debug import debug_print @@ -20,7 +20,7 @@ class GenerateGuardState(object): - def __init__(self, optimizer=None, guards=None, renum=None, bad=None): + def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False): self.optimizer = optimizer self.cpu = optimizer.cpu if guards is None: @@ -32,6 +32,7 @@ if bad is None: bad = {} self.bad = bad + self.force_boxes = force_boxes def get_runtime_item(self, box, descr, i): array = box.getref_base() @@ -303,7 +304,7 @@ opinfo = state.optimizer.getptrinfo(box) assert isinstance(opinfo, ArrayPtrInfo) else: - opinfo = None + opinfo = None for i in range(self.length): for descr in self.fielddescrs: index = i * len(self.fielddescrs) + descr.get_index() @@ -514,6 +515,8 @@ NotVirtualStateInfo.__init__(self, cpu, type, info) def _generate_guards(self, other, box, runtime_box, state): + if state.force_boxes and isinstance(other, VirtualStateInfo): + return self._generate_virtual_guards(other, box, runtime_box, state) if not isinstance(other, NotVirtualStateInfoPtr): raise VirtualStatesCantMatch( 'The VirtualStates does not match as a ' + @@ -545,6 +548,23 @@ # to an existing compiled loop or retracing the loop. Both alternatives # will always generate correct behaviour, but performance will differ. + def _generate_virtual_guards(self, other, box, runtime_box, state): + """ + Generate the guards and add state information for unifying a virtual + object with a non-virtual. This involves forcing the object in the + event that unification can succeed. Since virtual objects cannot be null, + this method need only check that the virtual object has the expected type. + """ + assert state.force_boxes and isinstance(other, VirtualStateInfo) + + if self.level == LEVEL_CONSTANT: + raise VirtualStatesCantMatch( + "cannot unify a constant value with a virtual object") + + if self.level == LEVEL_KNOWNCLASS: + if not self.known_class.same_constant(other.known_class): + raise VirtualStatesCantMatch("classes don't match") + def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards, state): if not isinstance(other, NotVirtualStateInfoPtr): @@ -617,10 +637,10 @@ return False return True - def generate_guards(self, other, boxes, runtime_boxes, optimizer): + def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False): assert (len(self.state) == len(other.state) == len(boxes) == len(runtime_boxes)) - state = GenerateGuardState(optimizer) + state = GenerateGuardState(optimizer, force_boxes=force_boxes) for i in range(len(self.state)): self.state[i].generate_guards(other.state[i], boxes[i], runtime_boxes[i], state) @@ -644,8 +664,8 @@ return boxes - def make_inputargs_and_virtuals(self, inputargs, optimizer): - inpargs = self.make_inputargs(inputargs, optimizer) + def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False): + inpargs = self.make_inputargs(inputargs, optimizer, force_boxes) # we append the virtuals here in case some stuff is proven # to be not a virtual and there are getfields in the short preamble # that will read items out of there @@ -653,7 +673,7 @@ for i in range(len(inputargs)): if not isinstance(self.state[i], NotVirtualStateInfo): virtuals.append(inputargs[i]) - + return inpargs, virtuals def debug_print(self, hdr='', bad=None, metainterp_sd=None): diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4507,3 +4507,54 @@ i += 1 return i self.meta_interp(f, []) + + def test_round_trip_raw_pointer(self): + # The goal of this test to to get a raw pointer op into the short preamble + # so we can check that the proper guards are generated + # In this case, the resulting short preamble contains + # + # i1 = getfield_gc_i(p0, descr=inst__ptr) + # i2 = int_eq(i1, 0) + # guard_false(i2) + # + # as opposed to what the JIT used to produce + # + # i1 = getfield_gc_i(p0, descr=inst__ptr) + # guard_nonnull(i1) + # + # Which will probably generate correct assembly, but the optimization + # pipline expects guard_nonnull arguments to be pointer ops and may crash + # and may crash on other input types. + driver = JitDriver(greens=[], reds=['i', 'val']) + + class Box(object): + _ptr = lltype.nullptr(rffi.CCHARP.TO) + + def new_int_buffer(value): + data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw') + rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value) + return data + + def read_int_buffer(buf): + return rffi.cast(rffi.INTP, buf)[0] + + def f(): + i = 0 + val = Box() + val._ptr = new_int_buffer(1) + + set_param(None, 'retrace_limit', -1) + while i < 100: + driver.jit_merge_point(i=i, val=val) + driver.can_enter_jit(i=i, val=val) + # Just to produce a side exit + if i & 0b100: + i += 1 + i += int(read_int_buffer(val._ptr)) + lltype.free(val._ptr, flavor='raw') + val._ptr = new_int_buffer(1) + lltype.free(val._ptr, flavor='raw') + + self.meta_interp(f, []) + self.check_resops(guard_nonnull=0) + diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1,8 +1,8 @@ import py -from rpython.rlib.jit import JitDriver, promote, dont_look_inside +from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param from rpython.rlib.objectmodel import compute_unique_id from rpython.jit.codewriter.policy import StopAtXPolicy -from rpython.jit.metainterp.test.support import LLJitMixin +from rpython.jit.metainterp.test.support import LLJitMixin, get_stats from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper import rclass from rpython.rtyper.lltypesystem.lloperation import llop @@ -965,6 +965,82 @@ self.check_aborted_count(0) self.check_target_token_count(4) + def test_avoid_preamble(self): + driver = JitDriver(greens=[], reds=['i', 'val']) + class X(object): + def __init__(self, v): + self.v = v + + class Box(object): + def __init__(self, v): + self.unbox = v + + mask = -2 + const = Box(X(5)) + def f(): + # Prevent all retracing of side exits. Ensures that the unroll + # optimizer will attempt to jump to either the preamble or loop. + set_param(driver, 'retrace_limit', -1) + set_param(driver, 'threshold', 1) + val = X(0) + i = 0 + const.unbox = X(5) + while i < 17: + driver.can_enter_jit(i=i, val=val) + driver.jit_merge_point(i=i, val=val) + # Logical & rather than comparison to confuse range analysis. + # Test only succeeds on the first 2 iterations + if i & -2 == 0: + val = const.unbox + else: + val = X(i) + i += 1 + return 0 + + self.meta_interp(f, []) + + # With retracing disable, there will be one optimized loop expecting a + # non-virtual X object. The side exit creates a virtual object which must + # be allocated to jump to the optimized trace. + self.check_resops(jump=3, label=2, new_with_vtable=2) + self.check_target_token_count(2) + self.check_trace_count(3) + + def test_conflated_virtual_states(self): + # All cases are covered when forcing one component of the virtual state + # also forces an as yet unseen component. + # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual + # objects + driver = JitDriver(greens=[], reds=['i', 'v1', 'v2']) + class Box(object): + def __init__(self, v): + self.v = v + + class X(object): + def __init__(self, v): + self.v = v + + const = Box(X(0)) + def f(): + set_param(None, 'retrace_limit', -1) + set_param(None, 'threshold', 1) + i = 0 + v1 = X(0) + v2 = X(0) + const.v = X(0) + while i < 17: + driver.jit_merge_point(i=i, v1=v1, v2=v2) + driver.can_enter_jit(i=i, v1=v1, v2=v2) + if i & 1 == 0: + v1 = const.v + v2 = X(i) + else: + v1 = v2 = X(i) + i += 1 + return None + self.meta_interp(f, []) + # assert did not crash + class VirtualMiscTests: def test_multiple_equal_virtuals(self): diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -97,7 +97,8 @@ OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"] HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000 -if OPENSSL_VERSION_NUMBER >= 0x10100000: +if (OPENSSL_VERSION_NUMBER >= 0x10100000 and + OPENSSL_VERSION_NUMBER < 0x20000000): # <= libressl :-( eci.pre_include_bits = () eci.post_include_bits = () raise Exception("""OpenSSL version >= 1.1 not supported yet. _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit