Author: Manuel Jacob <m...@manueljacob.de> Branch: py3.6 Changeset: r94007:a756aa082d6b Date: 2018-03-19 17:14 +0100 http://bitbucket.org/pypy/pypy/changeset/a756aa082d6b/
Log: hg merge py3.5 diff too long, truncating to 2000 out of 2599 lines diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -223,6 +223,7 @@ that fails it falls back to the VS90COMNTOOLS env var. """ vsbase = VS_BASE % version + batfile = 'vcvarsall.bat' try: productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, "productdir") @@ -235,9 +236,14 @@ toolsdir = os.environ.get(toolskey, None) if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) + if os.path.exists(os.path.join(toolsdir, 'VsDevCmd.bat')): + productdir = toolsdir + batfile = 'VsDevCmd.bat' + else: + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) return None else: @@ -245,7 +251,7 @@ if not productdir: log.debug("No productdir found") return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") + vcvarsall = os.path.join(productdir, batfile) if os.path.isfile(vcvarsall): return vcvarsall log.debug("Unable to find vcvarsall.bat") @@ -289,6 +295,7 @@ if len(result) != len(interesting): raise ValueError(str(list(result.keys()))) + log.debug('Got', result) return result # More globals diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py --- a/lib_pypy/_dbm.py +++ b/lib_pypy/_dbm.py @@ -168,7 +168,14 @@ def open(filename, flag='r', mode=0o666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") filename = filename.encode(sys.getdefaultencoding()) openflag = 0 diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -3,7 +3,7 @@ import importlib.machinery -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -23,10 +23,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -35,13 +53,12 @@ return suffixes[0] if suffixes else None -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.4 +Version: 1.11.5 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -896,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -143,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -164,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -567,6 +567,11 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in the + directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically ``VS140COMNTOOLS``) + environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,17 @@ .. branch: winapi Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -113,6 +113,7 @@ def test_pytrace(): output = run(sys.executable, pypypath, '-S', stdin="__pytrace__ = 1\nx = 5\nx") + output = output.replace('\r\n', '\n') assert ('\t<module>: LOAD_CONST 0 (5)\n' '\t<module>: STORE_NAME 0 (x)\n' '\t<module>: LOAD_CONST 1 (None)\n' diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.4" +VERSION = "1.11.5" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -53,8 +53,7 @@ self.libhandle = rffi.cast(DLLHANDLE, 0) if not libhandle: - raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed", - self.libname) + return self.may_unregister_rpython_finalizer(self.ffi.space) # Clear the dict to force further accesses to do cdlopen_fetch() diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -38,9 +38,16 @@ space = self.space return space.newtext("<clibrary '%s'>" % self.name) + def check_closed(self): + if self.handle == rffi.cast(DLLHANDLE, 0): + raise oefmt(self.space.w_ValueError, + "library '%s' has already been closed", + self.name) + @unwrap_spec(w_ctype=W_CType, name='text') def load_function(self, w_ctype, name): from pypy.module._cffi_backend import ctypeptr, ctypearray + self.check_closed() space = self.space # if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray): @@ -60,6 +67,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def read_variable(self, w_ctype, name): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -71,6 +79,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def write_variable(self, w_ctype, name, w_value): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -80,6 +89,9 @@ name, self.name) w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value) + def close_lib(self): + self._finalize_() + W_Library.typedef = TypeDef( '_cffi_backend.Library', @@ -87,6 +99,7 @@ load_function = interp2app(W_Library.load_function), read_variable = interp2app(W_Library.read_variable), write_variable = interp2app(W_Library.write_variable), + close_lib = interp2app(W_Library.close_lib), ) W_Library.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.5", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -395,6 +395,10 @@ # the next one is from 'libm', not 'libc', but we assume # that it is already loaded too, so it should work assert x.load_function(BVoidP, 'sqrt') + # + x.close_lib() + py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") @@ -1210,6 +1214,9 @@ ll = find_and_load_library('c') stderr = ll.read_variable(BVoidP, "stderr") assert stderr == cast(BVoidP, _testfunc(8)) + # + ll.close_lib() + py.test.raises(ValueError, ll.read_variable, BVoidP, "stderr") def test_read_variable_as_unknown_length_array(): ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard @@ -1236,6 +1243,9 @@ assert not ll.read_variable(BVoidP, "stderr") ll.write_variable(BVoidP, "stderr", stderr) assert ll.read_variable(BVoidP, "stderr") == stderr + # + ll.close_lib() + py.test.raises(ValueError, ll.write_variable, BVoidP, "stderr", stderr) def test_callback(): BInt = new_primitive_type("int") diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -114,12 +114,10 @@ from re_python_pysrc import ffi lib = ffi.dlopen(self.extmod) ffi.dlclose(lib) - e = raises(ffi.error, ffi.dlclose, lib) - assert str(e.value) == ( - "library '%s' is already closed" % (self.extmod,)) e = raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (self.extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(self): self.fix_path() diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py --- a/pypy/module/_posixsubprocess/test/test_subprocess.py +++ b/pypy/module/_posixsubprocess/test/test_subprocess.py @@ -1,4 +1,9 @@ from os.path import dirname +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") + class AppTestSubprocess: spaceconfig = dict(usemodules=('_posixsubprocess', 'signal', diff --git a/pypy/module/_posixsubprocess/test/test_ztranslation.py b/pypy/module/_posixsubprocess/test/test_ztranslation.py --- a/pypy/module/_posixsubprocess/test/test_ztranslation.py +++ b/pypy/module/_posixsubprocess/test/test_ztranslation.py @@ -1,4 +1,8 @@ from pypy.objspace.fake.checkmodule import checkmodule +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") def test_posixsubprocess_translates(): checkmodule('_posixsubprocess') diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -813,10 +813,10 @@ def test_recv_send_timeout(self): from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() + cli.settimeout(1.0) cli.connect(self.serv.getsockname()) fileno, addr = self.serv._accept() t = socket(fileno=fileno) - cli.settimeout(1.0) # test recv() timeout t.send(b'*') buf = cli.recv(100) diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -407,8 +407,11 @@ if not (last_pos == ctx.match_start == ctx.match_end and n > 0): # the above ignores empty matches on latest position + last_pos = ctx.match_end if filter_is_callable: w_match = self.getmatch(ctx, True) + # make a copy of 'ctx'; see test_sub_matches_stay_valid + ctx = ctx.fresh_copy(start) # match_start/match_end dropped w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None and unicodebuilder is None @@ -425,7 +428,6 @@ unicodebuilder.append(filter_as_unicode) else: sublist_w.append(w_filter) - last_pos = ctx.match_end n += 1 elif last_pos >= ctx.end: break # empty match at the end: finished diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -395,6 +395,18 @@ KEYCRE = re.compile(r"%\(([^)]*)\)s|.") raises(TypeError, KEYCRE.sub, "hello", {"%(": 1}) + def test_sub_matches_stay_valid(self): + import re + matches = [] + def callback(match): + matches.append(match) + return "x" + result = re.compile(r"[ab]").sub(callback, "acb") + assert result == "xcx" + assert len(matches) == 2 + assert matches[0].group() == "a" + assert matches[1].group() == "b" + class AppTestSreScanner: diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -175,7 +175,7 @@ c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + ret = rwinreg.RegLoadKeyW(hkey, c_subkey, c_filename) if ret != 0: raiseWindowsError(space, ret, 'RegLoadKey') @@ -196,7 +196,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegSaveKey(hkey, c_filename, None) + ret = rwinreg.RegSaveKeyW(hkey, c_filename, None) if ret != 0: raiseWindowsError(space, ret, 'RegSaveKey') @@ -226,7 +226,7 @@ c_subkey = rffi.cast(rffi.CCHARP, subkey) with rffi.scoped_unicode2wcharp(value) as dataptr: c_dataptr = rffi.cast(rffi.CCHARP, dataptr) - ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + ret = rwinreg.RegSetValueW(hkey, c_subkey, rwinreg.REG_SZ, c_dataptr, len(value)) if ret != 0: raiseWindowsError(space, ret, 'RegSetValue') @@ -250,7 +250,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, None, bufsize_p) bufSize = intmask(bufsize_p[0]) if ret == rwinreg.ERROR_MORE_DATA: bufSize = 256 @@ -259,7 +259,7 @@ while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, buf, bufsize_p) if ret == rwinreg.ERROR_MORE_DATA: print 'bufSize was %d, too small' % bufSize # Resize and retry @@ -440,7 +440,7 @@ try: with rffi.scoped_unicode2wcharp(value_name) as wide_vn: c_vn = rffi.cast(rffi.CCHARP, wide_vn) - ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) + ret = rwinreg.RegSetValueExW(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -460,7 +460,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, null_dword, None, retDataSize) bufSize = intmask(retDataSize[0]) if ret == rwinreg.ERROR_MORE_DATA: @@ -472,7 +472,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -505,7 +505,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + ret = rwinreg.RegCreateKeyW(hkey, c_subkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) @@ -527,7 +527,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + ret = rwinreg.RegCreateKeyExW(hkey, c_subkey, reserved, None, 0, access, None, rethkey, lltype.nullptr(rwin32.LPDWORD.TO)) if ret != 0: @@ -549,7 +549,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteKey(hkey, c_subkey) + ret = rwinreg.RegDeleteKeyW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteKey') @@ -562,7 +562,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteValue(hkey, c_subkey) + ret = rwinreg.RegDeleteValueW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') @@ -582,7 +582,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + ret = rwinreg.RegOpenKeyExW(hkey, c_subkey, reserved, access, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegOpenKeyEx') return W_HKEY(space, rethkey[0]) @@ -607,7 +607,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, null_dword, null_dword, null_dword, null_dword, retValueSize, retDataSize, @@ -628,7 +628,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) - ret = rwinreg.RegEnumValue( + ret = rwinreg.RegEnumValueW( hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: @@ -673,7 +673,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: retValueSize[0] = r_uint(257) # includes NULL terminator - ret = rwinreg.RegEnumKeyEx(hkey, index, buf, retValueSize, + ret = rwinreg.RegEnumKeyExW(hkey, index, buf, retValueSize, null_dword, None, null_dword, lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: @@ -695,7 +695,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as nValues: with lltype.scoped_alloc(rwin32.PFILETIME.TO, 1) as ft: null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, nSubKeys, null_dword, null_dword, nValues, null_dword, null_dword, @@ -722,7 +722,7 @@ machine = space.text_or_none_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) + ret = rwinreg.RegConnectRegistryW(machine, hkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegConnectRegistry') return W_HKEY(space, rethkey[0]) diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -207,18 +207,20 @@ except: pass - key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - SaveKey(key, self.tmpfilename) + with OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) as key: + SaveKey(key, self.tmpfilename) def test_expand_environment_string(self): from winreg import ExpandEnvironmentStrings import nt r = ExpandEnvironmentStrings("%windir%\\test") assert isinstance(r, str) - if 'WINDIR' in list(nt.environ.keys()): + if 'WINDIR' in nt.environ: assert r == nt.environ["WINDIR"] + "\\test" + elif 'windir' in nt.environ: + assert r == nt.environ["windir"] + "\\test" else: - assert r == nt.environ["windir"] + "\\test" + skip('nt.environ not filled in for untranslated tests') def test_long_key(self): from winreg import ( diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -189,33 +189,6 @@ decref(space, view.c_obj) return 0 -def fill_buffer(space, view, pybuf, py_obj): - view.c_buf = cts.cast('void *', pybuf.get_raw_address()) - view.c_obj = py_obj - if py_obj: - incref(space, py_obj) - view.c_len = pybuf.getlength() - view.c_itemsize = pybuf.getitemsize() - rffi.setintfield(view, 'c_readonly', int(pybuf.readonly)) - rffi.setintfield(view, 'c_ndim', pybuf.getndim()) - view.c_format = rffi.str2charp(pybuf.getformat()) - shape = pybuf.getshape() - if not shape: - view.c_shape = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_shape = cts.cast('Py_ssize_t*', view.c__shape) - for i, n in enumerate(shape): - view.c_shape[i] = n - strides = pybuf.getstrides() - if not strides: - view.c_strides = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_strides = cts.cast('Py_ssize_t*', view.c__strides) - for i, n in enumerate(strides): - view.c_strides[i] = n - view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) - view.c_internal = lltype.nullptr(rffi.VOIDP.TO) - DEFAULT_FMT = rffi.str2charp("B") @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -688,12 +688,46 @@ return space.call_args(space.get(new_fn, w_self), args) return slot_tp_new +@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], + rffi.INT_real, error=-1) +def bytes_getbuffer(space, w_str, view, flags): + from pypy.module.cpyext.bytesobject import PyBytes_AsString + from pypy.module.cpyext.buffer import PyBuffer_FillInfo + c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) + return PyBuffer_FillInfo(space, view, w_str, c_buf, + space.len_w(w_str), 1, flags) + +def slot_from_buffer_w(space, typedef): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, c_view, flags): + w_obj = w_self + if c_view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + c_view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + c_view.c_obj = make_ref(space, w_s) + c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(c_view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, c_view) + return ret + return 0 + return buff_w + @slot_factory('tp_as_buffer.c_bf_getbuffer') def make_bf_getbuffer(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - return slot_from___buffer__(space, typedef, buff_fn) + if space.is_w(w_type, space.w_bytes): + return bytes_getbuffer elif typedef.buffer: return slot_from_buffer_w(space, typedef) else: @@ -739,59 +773,6 @@ return slot_tp_descr_set -def slot_from___buffer__(space, typedef, buff_fn): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - -def slot_from_buffer_w(space, typedef): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - w_obj = w_self - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - missing_wrappers = ['wrap_indexargfunc', 'wrap_del'] for name in missing_wrappers: assert name not in globals() diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -640,6 +640,33 @@ self.attr1 = 123 assert module.test_tp_getattro(C(), 123) + def test_issue_2760_getattr(self): + module = self.import_extension('foo', [ + ("get_foo", "METH_O", + ''' + char* name = "foo"; + PyTypeObject *tp = Py_TYPE(args); + PyObject *res; + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(args, name); + } + else if (tp->tp_getattro != NULL) { + PyObject *w = PyUnicode_FromString(name); + res = (*tp->tp_getattro)(args, w); + Py_DECREF(w); + } + else { + res = Py_None; + } + return res; + ''')]) + class Passthrough(object): + def __getattr__(self, name): + return name + + obj = Passthrough() + assert module.get_foo(obj) == 'foo' + def test_nb_int(self): module = self.import_extension('foo', [ ("nb_int", "METH_VARARGS", diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -236,11 +236,6 @@ def update_all_slots(space, w_type, pto): # fill slots in pto - # Not very sure about it, but according to - # test_call_tp_dealloc, we should not - # overwrite slots that are already set: these ones are probably - # coming from a parent C type. - for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: slot_func_helper = None w_descr = w_type.dict_w.get(method_name, None) @@ -276,8 +271,7 @@ def fill_slot(space, pto, w_type, slot_names, slot_func_helper): # XXX special case wrapper-functions and use a "specific" slot func if len(slot_names) == 1: - if not getattr(pto, slot_names[0]): - setattr(pto, slot_names[0], slot_func_helper) + setattr(pto, slot_names[0], slot_func_helper) elif ((w_type is space.w_list or w_type is space.w_tuple) and slot_names[0] == 'c_tp_as_number'): # XXX hack - how can we generalize this? The problem is method @@ -312,8 +306,7 @@ struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True) setattr(pto, slot_names[0], struct) - if not getattr(struct, slot_names[1]): - setattr(struct, slot_names[1], slot_func_helper) + setattr(struct, slot_names[1], slot_func_helper) def add_operators(space, dict_w, pto, name): from pypy.module.cpyext.object import PyObject_HashNotImplemented @@ -526,33 +519,6 @@ realize=type_realize, dealloc=type_dealloc) -@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bytes_getbuffer(space, w_str, view, flags): - from pypy.module.cpyext.bytesobject import PyBytes_AsString - from pypy.module.cpyext.buffer import PyBuffer_FillInfo - c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) - return PyBuffer_FillInfo(space, view, w_str, c_buf, - space.len_w(w_str), 1, flags) - -@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bf_getbuffer(space, w_obj, view, flags): - from pypy.module.cpyext.buffer import fill_buffer - buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags)) - fill_buffer(space, view, buf, as_pyobj(space, w_obj)) - return 0 - -def setup_buffer_procs(space, w_type, pto): - bufspec = w_type.layout.typedef.buffer - if not bufspec: - return - c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - lltype.render_immortal(c_buf) - if space.is_w(w_type, space.w_bytes): - c_buf.c_bf_getbuffer = llslot(space, bytes_getbuffer) - else: - c_buf.c_bf_getbuffer = llslot(space, bf_getbuffer) - pto.c_tp_as_buffer = c_buf - @slot_function([PyObject], lltype.Void) def type_dealloc(space, obj): from pypy.module.cpyext.object import _dealloc @@ -611,8 +577,6 @@ pto.c_tp_itemsize = 1 elif space.is_w(w_type, space.w_tuple): pto.c_tp_itemsize = rffi.sizeof(PyObject) - # buffer protocol - setup_buffer_procs(space, w_type, pto) state = space.fromcache(State) pto.c_tp_free = state.C.PyObject_Free @@ -730,7 +694,6 @@ pto.c_tp_as_buffer = base.c_tp_as_buffer if base.c_tp_as_buffer: # inherit base.c_tp_as_buffer functions not inherited from w_type - # note: builtin types are handled in setup_buffer_procs pto_as = pto.c_tp_as_buffer base_as = base.c_tp_as_buffer if not pto_as.c_bf_getbuffer: diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -106,7 +106,7 @@ return space.getitem(w_obj1, w_obj2) @slot_function([PyObject, PyObject], PyObject) -def slot_tp_getattr(space, w_obj1, w_obj2): +def slot_tp_getattr_hook(space, w_obj1, w_obj2): return space.getattr(w_obj1, w_obj2) @slot_function([PyObject, PyObject, PyObject], PyObject) diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -82,7 +82,7 @@ import imp for suffix, mode, type in imp.get_suffixes(): if type == imp.PY_SOURCE: - assert suffix == '.py' + assert suffix in ('.py', '.pyw') assert mode == 'r' elif type == imp.PY_COMPILED: assert suffix == '.pyc' diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1092,9 +1092,11 @@ if sys.platform == 'win32': os.chmod(my_path, 0o400) assert (os.stat(my_path).st_mode & 0o600) == 0o400 + os.chmod(self.path, 0o700) else: os.chmod(my_path, 0o200) assert (os.stat(my_path).st_mode & 0o777) == 0o200 + os.chmod(self.path, 0o700) if hasattr(os, 'fchmod'): def test_fchmod(self): @@ -1406,6 +1408,22 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") + def test_rename(self): + os = self.posix + fname = self.path2 + 'rename.txt' + with open(fname, "w") as f: + f.write("this is a rename test") + unicode_name = str(self.udir) + u'/test\u03be.txt' + os.rename(fname, unicode_name) + with open(unicode_name) as f: + assert f.read() == 'this is a rename test' + os.rename(unicode_name, fname) + with open(fname) as f: + assert f.read() == 'this is a rename test' + os.unlink(fname) + + def test_device_encoding(self): import sys encoding = self.posix.device_encoding(sys.stdout.fileno()) @@ -1509,6 +1527,8 @@ def test_environ(self): import sys, os environ = os.environ + if not environ: + skip('environ not filled in for untranslated tests') for k, v in environ.items(): assert type(k) is str assert type(v) is str diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -500,3 +500,21 @@ """) m = ffi.dlopen(lib_m) assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc', 'myvar'] + + def test_dlclose(self): + if self.Backend is CTypesBackend: + py.test.skip("not with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef("int foobar(void); int foobaz;") + lib = ffi.dlopen(lib_m) + ffi.dlclose(lib) + e = py.test.raises(ValueError, getattr, lib, 'foobar') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobaz') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + ffi.dlclose(lib) # does not raise diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py @@ -115,8 +115,12 @@ if sys.platform == 'win32': import os # did we already build it? - if os.path.exists(str(udir.join('testownlib.dll'))): - cls.module = str(udir.join('testownlib.dll')) + if cls.Backend is CTypesBackend: + dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend + else: + dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') # non-ascii char + if os.path.exists(dll_path): + cls.module = dll_path return # try (not too hard) to find the version used to compile this python # no mingw @@ -136,8 +140,9 @@ if os.path.isfile(vcvarsall): cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ ' /LD /Fetestownlib.dll' - subprocess.check_call(cmd, cwd = str(udir), shell=True) - cls.module = str(udir.join('testownlib.dll')) + subprocess.check_call(cmd, cwd = str(udir), shell=True) + os.rename(str(udir) + '\\testownlib.dll', dll_path) + cls.module = dll_path else: subprocess.check_call( 'cc testownlib.c -shared -fPIC -o testownlib.so', diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py @@ -1,9 +1,10 @@ # Generated by pypy/tool/import_cffi.py -import sys +import sys, os import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from pypy.module.test_lib_pypy.cffi_tests.support import u def setup_module(mod): @@ -36,6 +37,13 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + if sys.platform == "win32": + # test with a non-ascii char + outputfn1 = outputfilename + ofn, oext = os.path.splitext(outputfn1) + outputfilename = ofn + (u+'\u03be') + oext + #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) + os.rename(outputfn1, outputfilename) mod.extmod = outputfilename mod.tmpdir = tmpdir # @@ -56,6 +64,9 @@ typedef struct bar_s { int x; signed char a[]; } bar_t; enum foo_e { AA, BB, CC }; int strlen(const char *); + struct with_union { union { int a; char b; }; }; + union with_struct { struct { int a; char b; }; }; + struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; }; """) ffi.set_source('re_python_pysrc', None) ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py'))) @@ -105,12 +116,14 @@ from re_python_pysrc import ffi lib = ffi.dlopen(extmod) ffi.dlclose(lib) - e = py.test.raises(ffi.error, ffi.dlclose, lib) - assert str(e.value).startswith( - "library '%s' is already closed" % (extmod,)) + if type(extmod) is not str: # unicode, on python 2 + str_extmod = extmod.encode('utf-8') + else: + str_extmod = extmod e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( - "library '%s' has been closed" % (extmod,)) + "library '%s' has been closed" % (str_extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(): from re_python_pysrc import ffi @@ -213,3 +226,23 @@ ffi.set_source('test_partial_enum', None) py.test.raises(VerificationMissing, ffi.emit_python_code, str(tmpdir.join('test_partial_enum.py'))) + +def test_anonymous_union_inside_struct(): + # based on issue #357 + from re_python_pysrc import ffi + INT = ffi.sizeof("int") + assert ffi.offsetof("struct with_union", "a") == 0 + assert ffi.offsetof("struct with_union", "b") == 0 + assert ffi.sizeof("struct with_union") == INT + # + assert ffi.offsetof("union with_struct", "a") == 0 + assert ffi.offsetof("union with_struct", "b") == INT + assert ffi.sizeof("union with_struct") >= INT + 1 + # + FLOAT = ffi.sizeof("float") + assert ffi.sizeof("struct NVGcolor") == FLOAT * 4 + assert ffi.offsetof("struct NVGcolor", "rgba") == 0 + assert ffi.offsetof("struct NVGcolor", "r") == 0 + assert ffi.offsetof("struct NVGcolor", "g") == FLOAT + assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2 + assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3 diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/pypy/module/test_lib_pypy/test_dbm_extra.py @@ -1,4 +1,4 @@ -import py +import py, os from rpython.tool.udir import udir try: from lib_pypy import dbm @@ -73,3 +73,8 @@ assert 'key_with_empty_value' in d assert d['key_with_empty_value'] == '' d.close() + +def test_unicode_filename(): + path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' + d = dbm.open(path, 'c') + d.close() diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,6 +1,7 @@ +import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -40,9 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 -@example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,13 +96,12 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: raise AssertionError("annotationoftype(%r)" % (t,)) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -354,7 +354,7 @@ else: # this is dead code, but in case we have a gc that does # not have a write barrier and does not zero memory, we would - # need to clal it + # need to call it if op.getopnum() == rop.SETFIELD_GC: self.consider_setfield_gc(op) elif op.getopnum() == rop.SETARRAYITEM_GC: diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -30,7 +30,7 @@ class CompileData(object): memo = None log_noopt = True - + def forget_optimization_info(self): for arg in self.trace.inputargs: arg.set_forwarded(None) @@ -67,19 +67,26 @@ """ This represents label() ops jump with no extra info associated with the label """ - def __init__(self, trace, call_pure_results=None, + def __init__(self, trace, resumestorage=None, call_pure_results=None, enable_opts=None): self.trace = trace + self.resumestorage = resumestorage self.call_pure_results = call_pure_results self.enable_opts = enable_opts def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer + from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace.get_iter(), - self.call_pure_results) + traceiter = self.trace.get_iter() + if self.resumestorage: + frontend_inputargs = self.trace.inputargs + deserialize_optimizer_knowledge(opt, self.resumestorage, + frontend_inputargs, + traceiter.inputargs) + return opt.propagate_all_forward(traceiter, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some @@ -518,7 +525,7 @@ for item in lst: item.set_forwarded(None) # XXX we should really do it, but we need to remember the values - # somehoe for ContinueRunningNormally + # somehow for ContinueRunningNormally if reset_values: item.reset_value() @@ -671,38 +678,16 @@ raise jitexc.ExitFrameWithExceptionRef(cpu, value) -class TerminatingLoopToken(JitCellToken): # FIXME: kill? - terminating = True - - def __init__(self, nargs, finishdescr): - self.finishdescr = finishdescr - -def make_done_loop_tokens(): - done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() - done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() - done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() - done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() - exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() - - # pseudo loop tokens to make the life of optimize.py easier - d = {'loop_tokens_done_with_this_frame_int': [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ], - 'loop_tokens_done_with_this_frame_ref': [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ], - 'loop_tokens_done_with_this_frame_float': [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ], - 'loop_tokens_done_with_this_frame_void': [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ], - 'loop_tokens_exit_frame_with_exception_ref': [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ], - } - d.update(locals()) - return d +def make_and_attach_done_descrs(targets): + for name, cls in [ + ("done_with_this_frame_descr_void", DoneWithThisFrameDescrVoid), + ("done_with_this_frame_descr_int", DoneWithThisFrameDescrInt), + ("done_with_this_frame_descr_ref", DoneWithThisFrameDescrRef), + ("done_with_this_frame_descr_float", DoneWithThisFrameDescrFloat), + ("exit_frame_with_exception_descr_ref", ExitFrameWithExceptionDescrRef)]: + descr = cls() + for target in targets: + setattr(target, name, descr) class ResumeDescr(AbstractFailDescr): _attrs_ = () @@ -726,6 +711,9 @@ TY_REF = 0x04 TY_FLOAT = 0x06 + def get_resumestorage(self): + raise NotImplementedError("abstract base class") + def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd) and not rstack.stack_almost_full()): @@ -854,15 +842,23 @@ class ResumeGuardCopiedDescr(AbstractResumeGuardDescr): _attrs_ = ('status', 'prev') + def __init__(self, prev): + AbstractResumeGuardDescr.__init__(self) + assert isinstance(prev, ResumeGuardDescr) + self.prev = prev + def copy_all_attributes_from(self, other): assert isinstance(other, ResumeGuardCopiedDescr) self.prev = other.prev def clone(self): - cloned = ResumeGuardCopiedDescr() - cloned.copy_all_attributes_from(self) + cloned = ResumeGuardCopiedDescr(self.prev) return cloned + def get_resumestorage(self): + prev = self.prev + assert isinstance(prev, ResumeGuardDescr) + return prev class ResumeGuardDescr(AbstractResumeGuardDescr): _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', @@ -873,8 +869,7 @@ rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) def copy_all_attributes_from(self, other): - if isinstance(other, ResumeGuardCopiedDescr): - other = other.prev + other = other.get_resumestorage() assert isinstance(other, ResumeGuardDescr) self.rd_consts = other.rd_consts self.rd_pendingfields = other.rd_pendingfields @@ -895,6 +890,9 @@ cloned.copy_all_attributes_from(self) return cloned + def get_resumestorage(self): + return self + class ResumeGuardExcDescr(ResumeGuardDescr): pass @@ -936,22 +934,22 @@ ptr = cpu.ts.cast_to_baseclass(gcref) return cast_base_ptr_to_instance(AllVirtuals, ptr) -def invent_fail_descr_for_op(opnum, optimizer, copied_guard=False): +def invent_fail_descr_for_op(opnum, optimizer, copied_from_descr=None): if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - assert not copied_guard + assert copied_from_descr is None resumedescr = ResumeGuardForcedDescr() resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd) elif opnum in (rop.GUARD_IS_OBJECT, rop.GUARD_SUBCLASS, rop.GUARD_GC_TYPE): # note - this only happens in tests resumedescr = ResumeAtPositionDescr() elif opnum in (rop.GUARD_EXCEPTION, rop.GUARD_NO_EXCEPTION): - if copied_guard: - resumedescr = ResumeGuardCopiedExcDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedExcDescr(copied_from_descr) else: resumedescr = ResumeGuardExcDescr() else: - if copied_guard: - resumedescr = ResumeGuardCopiedDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedDescr(copied_from_descr) else: resumedescr = ResumeGuardDescr() return resumedescr @@ -1036,6 +1034,9 @@ self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + def get_resumestorage(self): + return None + def compile_trace(metainterp, resumekey, runtime_boxes): """Try to compile a new bridge leading from the beginning of the history @@ -1067,22 +1068,15 @@ enable_opts = jitdriver_sd.warmstate.enable_opts call_pure_results = metainterp.call_pure_results + resumestorage = resumekey.get_resumestorage() if metainterp.history.ends_with_jump: - if isinstance(resumekey, ResumeGuardCopiedDescr): - key = resumekey.prev - assert isinstance(key, ResumeGuardDescr) - elif isinstance(resumekey, ResumeFromInterpDescr): - key = None - else: - key = resumekey - assert isinstance(key, ResumeGuardDescr) - data = BridgeCompileData(trace, runtime_boxes, key, + data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, inline_short_preamble=inline_short_preamble) else: - data = SimpleCompileData(trace, + data = SimpleCompileData(trace, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts) try: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -404,7 +404,6 @@ target_tokens = None failed_states = None retraced_count = 0 - terminating = False # see TerminatingLoopToken in compile.py invalidated = False outermost_jitdriver_sd = None # and more data specified by the backend when the loop is compiled @@ -935,7 +934,7 @@ return insns def check_simple_loop(self, expected=None, **check): - """ Usefull in the simplest case when we have only one trace ending with + """ Useful in the simplest case when we have only one trace ending with a jump back to itself and possibly a few bridges. Only the operations within the loop formed by that single jump will be counted. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # <length> # (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # <length> # (<box1> <index> <descr> <box2>) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# <length> +# (<const> <box2>) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -688,12 +688,10 @@ def _copy_resume_data_from(self, guard_op, last_guard_op): - descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, True) last_descr = last_guard_op.getdescr() + descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) assert isinstance(last_descr, compile.ResumeGuardDescr) - if isinstance(descr, compile.ResumeGuardCopiedDescr): - descr.prev = last_descr - else: + if not isinstance(descr, compile.ResumeGuardCopiedDescr): descr.copy_all_attributes_from(last_descr) guard_op.setdescr(descr) guard_op.setfailargs(last_guard_op.getfailargs()) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -31,8 +31,8 @@ expected = convert_old_style_to_targets(exp, jump=True) call_pure_results = self._convert_call_pure_results(call_pure_results) trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu)) - compile_data = compile.SimpleCompileData(trace, - call_pure_results) + compile_data = compile.SimpleCompileData( + trace, call_pure_results=call_pure_results) info, ops = self._do_optimize_loop(compile_data) label_op = ResOperation(rop.LABEL, info.inputargs) loop.inputargs = info.inputargs diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1854,12 +1854,7 @@ self._addr2name_keys = [] self._addr2name_values = [] - self.__dict__.update(compile.make_done_loop_tokens()) - for val in ['int', 'float', 'ref', 'void']: - fullname = 'done_with_this_frame_descr_' + val - setattr(self.cpu, fullname, getattr(self, fullname)) - d = self.exit_frame_with_exception_descr_ref - self.cpu.exit_frame_with_exception_descr_ref = d + compile.make_and_attach_done_descrs([self, cpu]) def _freeze_(self): return True @@ -1909,8 +1904,8 @@ history.REF: 'ref', history.FLOAT: 'float', history.VOID: 'void'}[jd.result_type] - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - jd.portal_finishtoken = tokens[0].finishdescr + token = getattr(self, 'done_with_this_frame_descr_%s' % name) + jd.portal_finishtoken = token jd.propagate_exc_descr = exc_descr # self.cpu.propagate_exception_descr = exc_descr @@ -2463,10 +2458,7 @@ def handle_guard_failure(self, resumedescr, deadframe): debug_start('jit-tracing') self.staticdata.profiler.start_tracing() - if isinstance(resumedescr, compile.ResumeGuardCopiedDescr): - key = resumedescr.prev - else: - key = resumedescr + key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp @@ -2770,21 +2762,19 @@ if result_type == history.VOID: assert exitbox is None exits = [] - loop_tokens = sd.loop_tokens_done_with_this_frame_void + token = sd.done_with_this_frame_descr_void elif result_type == history.INT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_int + token = sd.done_with_this_frame_descr_int elif result_type == history.REF: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_ref + token = sd.done_with_this_frame_descr_ref elif result_type == history.FLOAT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_float + token = sd.done_with_this_frame_descr_float else: assert False - # FIXME: kill TerminatingLoopToken? # FIXME: can we call compile_trace? - token = loop_tokens[0].finishdescr self.history.record(rop.FINISH, exits, None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, exits) @@ -2810,7 +2800,7 @@ def compile_exit_frame_with_exception(self, valuebox): self.store_token_in_vable() sd = self.staticdata - token = sd.loop_tokens_exit_frame_with_exception_ref[0].finishdescr + token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit