Author: Spenser Andrew Bauman <saba...@gmail.com> Branch: force-virtual-state Changeset: r86826:5649730037a2 Date: 2016-09-01 15:17 -0400 http://bitbucket.org/pypy/pypy/changeset/5649730037a2/
Log: Merge diff too long, truncating to 2000 out of 213792 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -27,3 +27,6 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -74,6 +74,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -117,7 +118,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -141,6 +141,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -211,6 +212,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + p_ziesch...@yahoo.de Attila Gobi Jasper.Schulz Christopher Pope @@ -221,6 +223,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -229,12 +232,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -355,115 +360,12 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz - - Heinrich-Heine University, Germany - Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany - Impara, Germany - Change Maker, Sweden - University of California Berkeley, USA - Google Inc. - King's College London - -The PyPy Logo as used by http://speed.pypy.org and others was created -by Samuel Reis and is distributed on terms of Creative Commons Share Alike -License. - -License for 'lib-python/2.7' -============================ - -Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' -directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html - -License for 'pypy/module/unicodedata/' -====================================== - -The following files are from the website of The Unicode Consortium -at http://www.unicode.org/. For the terms of use of these files, see -http://www.unicode.org/terms_of_use.html . Or they are derived from -files from the above website, and the same terms of use apply. - - CompositionExclusions-*.txt - EastAsianWidth-*.txt - LineBreak-*.txt - UnicodeData-*.txt - UnihanNumeric-*.txt - -License for 'dotviewer/font/' -============================= - -Copyright (C) 2008 The Android Open Source Project - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Detailed license information is contained in the NOTICE file in the -directory. - - -Licenses and Acknowledgements for Incorporated Software -======================================================= - -This section is an incomplete, but growing list of licenses and -acknowledgements for third-party software incorporated in the PyPy -distribution. - -License for 'Tcl/Tk' --------------------- - -This copy of PyPy contains library code that may, when used, result in -the Tcl/Tk library to be loaded. PyPy also includes code that may be -regarded as being a copy of some parts of the Tcl/Tk header files. -You may see a copy of the License for Tcl/Tk in the file -`lib_pypy/_tkinter/license.terms` included here. - -License for 'bzip2' -------------------- - -This copy of PyPy may be linked (dynamically or statically) with the -bzip2 library. You may see a copy of the License for bzip2/libbzip2 at - - http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html - -License for 'openssl' ---------------------- - -This copy of PyPy may be linked (dynamically or statically) with the -openssl library. You may see a copy of the License for OpenSSL at - - https://www.openssl.org/source/license.html - -License for 'gdbm' ------------------- - -The gdbm module includes code from gdbm.h, which is distributed under -the terms of the GPL license version 2 or any later version. Thus the -gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed -under the terms of the GPL license as well. - -License for 'rpython/rlib/rvmprof/src' --------------------------------------- - -The code is based on gperftools. You may see a copy of the License for it at - - https://github.com/gperftools/gperftools/blob/master/COPYING + werat diff --git a/_pytest/python.py b/_pytest/python.py --- a/_pytest/python.py +++ b/_pytest/python.py @@ -498,7 +498,10 @@ """ Collector for test methods. """ def collect(self): if hasinit(self.obj): - pytest.skip("class %s.%s with __init__ won't get collected" % ( + # XXX used to be skip(), but silently skipping classes + # XXX just because they have been written long ago is + # XXX imho a very, very, very bad idea + pytest.fail("class %s.%s with __init__ won't get collected" % ( self.obj.__module__, self.obj.__name__, )) diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -122,22 +122,24 @@ """Dummy method to let some easy_install packages that have optional C speedup components. """ + def customize(executable, flags): + command = compiler.executables[executable] + flags + setattr(compiler, executable, command) + if compiler.compiler_type == "unix": compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit']) compiler.shared_lib_extension = get_config_var('SO') if "CPPFLAGS" in os.environ: cppflags = shlex.split(os.environ["CPPFLAGS"]) - compiler.compiler.extend(cppflags) - compiler.compiler_so.extend(cppflags) - compiler.linker_so.extend(cppflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cppflags) if "CFLAGS" in os.environ: cflags = shlex.split(os.environ["CFLAGS"]) - compiler.compiler.extend(cflags) - compiler.compiler_so.extend(cflags) - compiler.linker_so.extend(cflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cflags) if "LDFLAGS" in os.environ: ldflags = shlex.split(os.environ["LDFLAGS"]) - compiler.linker_so.extend(ldflags) + customize('linker_so', ldflags) from sysconfig_cpython import ( diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -167,7 +167,7 @@ else: return self.value - def __buffer__(self): + def __buffer__(self, flags): return buffer(self._buffer) def _get_b_base(self): diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -342,7 +342,7 @@ thisarg = cast(thisvalue, POINTER(POINTER(c_void_p))) keepalives, newargs, argtypes, outargs, errcheckargs = ( self._convert_args(argtypes, args[1:], kwargs)) - newargs.insert(0, thisvalue.value) + newargs.insert(0, thisarg) argtypes.insert(0, c_void_p) else: thisarg = None diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -515,7 +515,7 @@ tovar, errcode) return # - elif isinstance(tp, (model.StructOrUnion, model.EnumType)): + elif isinstance(tp, model.StructOrUnionOrEnum): # a struct (not a struct pointer) as a function argument self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' % (tovar, self._gettypenum(tp), fromvar)) @@ -572,7 +572,7 @@ elif isinstance(tp, model.ArrayType): return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(model.PointerType(tp.item))) - elif isinstance(tp, model.StructType): + elif isinstance(tp, model.StructOrUnion): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( tp._get_c_name(), context)) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -308,7 +308,7 @@ elif isinstance(tp, model.ArrayType): return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(model.PointerType(tp.item))) - elif isinstance(tp, model.StructType): + elif isinstance(tp, model.StructOrUnion): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( tp._get_c_name(), context)) diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py --- a/lib_pypy/gdbm.py +++ b/lib_pypy/gdbm.py @@ -137,6 +137,8 @@ lib.gdbm_sync(self.__ll_dbm) def open(filename, flags='r', mode=0666): + if isinstance(filename, unicode): + filename = filename.encode() if flags[0] == 'r': iflags = lib.GDBM_READER elif flags[0] == 'w': diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -58,16 +58,16 @@ # General information about the project. project = u'PyPy' -copyright = u'2015, The PyPy Project' +copyright = u'2016, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '4.0' +version = '5.4' # The full version, including alpha/beta/rc tags. -release = '4.0.0' +release = '5.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -44,6 +44,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -87,7 +88,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -111,6 +111,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -181,6 +182,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + p_ziesch...@yahoo.de Attila Gobi Jasper.Schulz Christopher Pope @@ -191,6 +193,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -199,12 +202,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -325,9 +330,12 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz + werat diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.4.0.rst release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst release-5.1.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.4.0.rst whatsnew-pypy2-5.3.1.rst whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -57,7 +57,7 @@ -------------- Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. -Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy +Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy test suite. We could use help analyzing the failures and fixing them either as patches to upstream NumPy, or as fixes to PyPy. diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.4.0.rst @@ -0,0 +1,218 @@ +============ +PyPy2.7 v5.4 +============ + +We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3. +This new PyPy2.7 release includes incremental improvements to our C-API +compatability layer (cpyext), enabling us to pass over 99% of the upstream +numpy `test suite`_. We updated built-in cffi_ support to version 1.8, +which now supports the "limited API" mode for c-extensions on +CPython >=3.2. + +We improved tooling for the PyPy JIT_, and expanded VMProf +support to OpenBSD and Dragon Fly BSD + +As always, this release fixed many issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the PyPy2.7 v5.4 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility +.. _cffi: https://cffi.readthedocs.org +.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.3 released in June 2016) +========================================================= + +* New features: + + * Add `sys.{get,set}dlopenflags` + + * Improve CPython compatibility of 'is' for small and empty strings + + * Support for rgc.FinalizerQueue in the Boehm garbage collector + + * (RPython) support spawnv() if it is called in C `_spawnv` on windows + + * Fill in more slots when creating a PyTypeObject from a W_TypeObject, + like `__hex__`, `__sub__`, `__pow__` + + * Copy CPython's logic more closely for `isinstance()` and + `issubclass()` as well as `type.__instancecheck__()` and + `type.__subclasscheck__()` + + * Expose the name of CDLL objects + + * Rewrite the win32 dependencies of `subprocess` to use cffi + instead of ctypes + + * Improve the `JIT logging`_ facitilities + + * (RPython) make int * string work + + * Allocate all RPython strings with one extra byte, normally + unused. This now allows `ffi.from_buffer(string)` in CFFI with + no copy + + * Adds a new commandline option `-X track-resources` that will + produce a `ResourceWarning` when the GC closes a file or socket. + The traceback for the place where the file or socket was allocated + is given as well, which aids finding places where `close()` is + missing + + * Add missing `PyObject_Realloc`, `PySequence_GetSlice` + + * `type.__dict__` now returns a `dict_proxy` object, like on CPython. + Previously it returned what looked like a regular dict object (but + it was already read-only) + + * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5 + + * (RPython) add `rposix_scandir` portably, needed for Python 3.5 + + * Increased but incomplete support for memoryview attributes (format, + itemsize, ...) which also adds support for `PyMemoryView_FromObject` + +* Bug Fixes + + * Reject `mkdir()` in read-only sandbox filesystems + + * Add include guards to pymem.h to enable c++ compilation + + * Fix build breakage on OpenBSD and FreeBSD + + * Support OpenBSD, Dragon Fly BSD in VMProf + + * Fix for `bytearray('').replace('a', 'ab')` for empty strings + + * Sync internal state before calling `PyFile_AsFile()` + + * Allow writing to a char* from `PyString_AsString()` until it is + forced, also refactor `PyStringObject` to look like CPython's + and allow subclassing `PyString_Type` and `PyUnicode_Type` + + * Rpython rffi's socket(2) wrapper did not preserve errno + + * Refactor `PyTupleObject` to look like CPython's and allow + subclassing `PyTuple_Type` + + * Allow c-level assignment to a function pointer in a C-API + user-defined type after calling PyTypeReady by retrieving + a pointer to the function via offsets + rather than storing the function pointer itself + + * Use `madvise(MADV_FREE)`, or if that doesn't exist + `MADV_DONTNEED` on freed arenas to release memory back to the + OS for resource monitoring + + * Fix overflow detection in conversion of float to 64-bit integer + in timeout argument to various thread/threading primitives + + * Fix win32 outputting `\r\r\n` in some cases + + * Make `hash(-1)` return -2, as CPython does, and fix all the + ancilary places this matters + + * Fix `PyNumber_Check()` to behave more like CPython + + * (VMProf) Try hard to not miss any Python-level frame in the + captured stacks, even if there is metainterp or blackhole interp + involved. Also fix the stacklet (greenlet) support + + * Fix a critical JIT bug where `raw_malloc` -equivalent functions + lost the additional flags + + * Fix the mapdict cache for subclasses of builtin types that + provide a dict + + * Issues reported with our previous release were resolved_ after + reports from users on our issue tracker at + https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy + +* Performance improvements: + + * Add a before_call()-like equivalent before a few operations like + `malloc_nursery`, to move values from registers into other registers + instead of to the stack. + + * More tightly pack the stack when calling with `release gil` + + * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently + and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as + explicit interfaces. Clarify that `int_floordiv()` does python-style + rounding, unlike `llop.int_floordiv()`. + + * Use `ll_assert` (more often) in incminimark + + * (Testing) Simplify handling of interp-level tests and make it + more forward-compatible. Don't use interp-level RPython + machinery to test building app-level extensions in cpyext + + * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi + backend + + * Avoid a case in the JIT, where successive guard failures in + the same Python function end up as successive levels of + RPython functions, eventually exhausting the stack, while at + app-level the traceback is very short + + * Check for NULL returns from calls to the raw-malloc and raise, + rather than a guard + + * Improve `socket.recvfrom()` so that it copies less if possible + + * When generating C code, inline `goto` to blocks with only one + predecessor, generating less lines of code + + * When running the final backend-optimization phase before emitting + C code, constant-fold calls to we_are_jitted to return False. This + makes the generated C code a few percent smaller + + * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in + `interp_posix.py`, based on the clean-up of CPython 2.7.x + +.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,146 +1,9 @@ ========================== -What's new in PyPy2.7 5.3+ +What's new in PyPy2.7 5.4+ ========================== -.. this is a revision shortly after release-pypy2.7-v5.3 -.. startrev: 873218a739f1 +.. this is a revision shortly after release-pypy2.7-v5.4 +.. startrev: 522736f816dc -.. 418b05f95db5 -Improve CPython compatibility for ``is``. Now code like ``if x is ():`` -works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id . - -.. pull request #455 -Add sys.{get,set}dlopenflags, for cpyext extensions. - -.. branch: fix-gen-dfa - -Resolves an issue with the generator script to build the dfa for Python syntax. - -.. branch: z196-support - -Fixes a critical issue in the register allocator and extends support on s390x. -PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental) -and z196 (released August 2010) in addition to zEC12 and z13. -To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. - -.. branch: s390x-5.3-catchup - -Implement the backend related changes for s390x. - -.. branch: incminimark-ll_assert -.. branch: vmprof-openbsd - -.. branch: testing-cleanup - -Simplify handling of interp-level tests and make it more forward- -compatible. - -.. branch: pyfile-tell -Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile - -.. branch: rw-PyString_AS_STRING -Allow rw access to the char* returned from PyString_AS_STRING, also refactor -PyStringObject to look like cpython's and allow subclassing PyString_Type and -PyUnicode_Type - -.. branch: save_socket_errno - -Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show -the errno of the failing system call, but instead some random previous -errno. - -.. branch: PyTuple_Type-subclass - -Refactor PyTupleObject to look like cpython's and allow subclassing -PyTuple_Type - -.. branch: call-via-pyobj - -Use offsets from PyTypeObject to find actual c function to call rather than -fixed functions, allows function override after PyType_Ready is called - -.. branch: issue2335 - -Avoid exhausting the stack in the JIT due to successive guard -failures in the same Python function ending up as successive levels of -RPython functions, while at app-level the traceback is very short - -.. branch: use-madv-free - -Try harder to memory to the OS. See e.g. issue #2336. Note that it does -not show up as a reduction of the VIRT column in ``top``, and the RES -column might also not show the reduction, particularly on Linux >= 4.5 or -on OS/X: it uses MADV_FREE, which only marks the pages as returnable to -the OS if the memory is low. - -.. branch: cpyext-slotdefs2 - -Fill in more slots when creating a PyTypeObject from a W_TypeObject -More slots are still TBD, like tp_print and richcmp - -.. branch: json-surrogates - -Align json module decode with the cpython's impl, fixes issue 2345 - -.. branch: issue2343 - -Copy CPython's logic more closely for handling of ``__instancecheck__()`` -and ``__subclasscheck__()``. Fixes issue 2343. - -.. branch: msvcrt-cffi - -Rewrite the Win32 dependencies of 'subprocess' to use cffi instead -of ctypes. This avoids importing ctypes in many small programs and -scripts, which in turn avoids enabling threads (because ctypes -creates callbacks at import time, and callbacks need threads). - -.. branch: new-jit-log - -The new logging facility that integrates with and adds features to vmprof.com. - -.. branch: jitlog-32bit - -Resolve issues to use the new logging facility on a 32bit system - -.. branch: ep2016sprint - -Trying harder to make hash(-1) return -2, like it does on CPython - -.. branch: jitlog-exact-source-lines - -Log exact line positions in debug merge points. - -.. branch: null_byte_after_str - -Allocate all RPython strings with one extra byte, normally unused. -It is used to hold a final zero in case we need some ``char *`` -representation of the string, together with checks like ``not -can_move()`` or object pinning. Main new thing that this allows: -``ffi.from_buffer(string)`` in CFFI. Additionally, and most -importantly, CFFI calls that take directly a string as argument don't -copy the string any more---this is like CFFI on CPython. - -.. branch: resource_warning - -Add a new command line option -X track-resources which will produce -ResourceWarnings when the GC closes unclosed files and sockets. - -.. branch: cpyext-realloc - -Implement PyObject_Realloc - -.. branch: inline-blocks - -Improve a little bit the readability of the generated C code - -.. branch: improve-vmprof-testing - -Improved vmprof support: now tries hard to not miss any Python-level -frame in the captured stacks, even if there is the metainterp or -blackhole interp involved. Also fix the stacklet (greenlet) support. - -.. branch: py2-mappingproxy - -``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython. -Previously it returned what looked like a regular dict object (but it -was already read-only). +.. branch: rpython-resync +Backport rpython changes made directly on the py3k and py3.5 branches. diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst @@ -0,0 +1,165 @@ +========================= +What's new in PyPy2.7 5.4 +========================= + +.. this is a revision shortly after release-pypy2.7-v5.3 +.. startrev: 873218a739f1 + +.. 418b05f95db5 +Improve CPython compatibility for ``is``. Now code like ``if x is ():`` +works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id . + +.. pull request #455 +Add sys.{get,set}dlopenflags, for cpyext extensions. + +.. branch: fix-gen-dfa + +Resolves an issue with the generator script to build the dfa for Python syntax. + +.. branch: z196-support + +Fixes a critical issue in the register allocator and extends support on s390x. +PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental) +and z196 (released August 2010) in addition to zEC12 and z13. +To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment. + +.. branch: s390x-5.3-catchup + +Implement the backend related changes for s390x. + +.. branch: incminimark-ll_assert +.. branch: vmprof-openbsd + +.. branch: testing-cleanup + +Simplify handling of interp-level tests and make it more forward- +compatible. + +.. branch: pyfile-tell +Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile + +.. branch: rw-PyString_AS_STRING +Allow rw access to the char* returned from PyString_AS_STRING, also refactor +PyStringObject to look like cpython's and allow subclassing PyString_Type and +PyUnicode_Type + +.. branch: save_socket_errno + +Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show +the errno of the failing system call, but instead some random previous +errno. + +.. branch: PyTuple_Type-subclass + +Refactor PyTupleObject to look like cpython's and allow subclassing +PyTuple_Type + +.. branch: call-via-pyobj + +Use offsets from PyTypeObject to find actual c function to call rather than +fixed functions, allows function override after PyType_Ready is called + +.. branch: issue2335 + +Avoid exhausting the stack in the JIT due to successive guard +failures in the same Python function ending up as successive levels of +RPython functions, while at app-level the traceback is very short + +.. branch: use-madv-free + +Try harder to memory to the OS. See e.g. issue #2336. Note that it does +not show up as a reduction of the VIRT column in ``top``, and the RES +column might also not show the reduction, particularly on Linux >= 4.5 or +on OS/X: it uses MADV_FREE, which only marks the pages as returnable to +the OS if the memory is low. + +.. branch: cpyext-slotdefs2 + +Fill in more slots when creating a PyTypeObject from a W_TypeObject +More slots are still TBD, like tp_print and richcmp + +.. branch: json-surrogates + +Align json module decode with the cpython's impl, fixes issue 2345 + +.. branch: issue2343 + +Copy CPython's logic more closely for handling of ``__instancecheck__()`` +and ``__subclasscheck__()``. Fixes issue 2343. + +.. branch: msvcrt-cffi + +Rewrite the Win32 dependencies of 'subprocess' to use cffi instead +of ctypes. This avoids importing ctypes in many small programs and +scripts, which in turn avoids enabling threads (because ctypes +creates callbacks at import time, and callbacks need threads). + +.. branch: new-jit-log + +The new logging facility that integrates with and adds features to vmprof.com. + +.. branch: jitlog-32bit + +Resolve issues to use the new logging facility on a 32bit system + +.. branch: ep2016sprint + +Trying harder to make hash(-1) return -2, like it does on CPython + +.. branch: jitlog-exact-source-lines + +Log exact line positions in debug merge points. + +.. branch: null_byte_after_str + +Allocate all RPython strings with one extra byte, normally unused. +It is used to hold a final zero in case we need some ``char *`` +representation of the string, together with checks like ``not +can_move()`` or object pinning. Main new thing that this allows: +``ffi.from_buffer(string)`` in CFFI. Additionally, and most +importantly, CFFI calls that take directly a string as argument don't +copy the string any more---this is like CFFI on CPython. + +.. branch: resource_warning + +Add a new command line option -X track-resources which will produce +ResourceWarnings when the GC closes unclosed files and sockets. + +.. branch: cpyext-realloc + +Implement PyObject_Realloc + +.. branch: inline-blocks + +Improve a little bit the readability of the generated C code + +.. branch: improve-vmprof-testing + +Improved vmprof support: now tries hard to not miss any Python-level +frame in the captured stacks, even if there is the metainterp or +blackhole interp involved. Also fix the stacklet (greenlet) support. + +.. branch: py2-mappingproxy + +``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython. +Previously it returned what looked like a regular dict object (but it +was already read-only). + + +.. branch: const-fold-we-are-jitted + +Reduce the size of the generated C code by constant-folding ``we_are_jitted`` +in non-jitcode. + +.. branch: memoryview-attributes + +Support for memoryview attributes (format, itemsize, ...). +Extends the cpyext emulation layer. + +.. branch: redirect-assembler-jitlog + +Log more information to properly rebuild the redirected traces in jitviewer. + +.. branch: cpyext-subclass + +Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,7 +208,8 @@ def buffer_w(self, space, flags): w_impl = space.lookup(self, '__buffer__') if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) + w_result = space.get_and_call_function(w_impl, self, + space.newint(flags)) if space.isinstance_w(w_result, space.w_buffer): return w_result.buffer_w(space, flags) raise BufferInterfaceNotFound @@ -216,7 +217,8 @@ def readbuf_w(self, space): w_impl = space.lookup(self, '__buffer__') if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) + w_result = space.get_and_call_function(w_impl, self, + space.newint(space.BUF_FULL_RO)) if space.isinstance_w(w_result, space.w_buffer): return w_result.readbuf_w(space) raise BufferInterfaceNotFound @@ -224,7 +226,8 @@ def writebuf_w(self, space): w_impl = space.lookup(self, '__buffer__') if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) + w_result = space.get_and_call_function(w_impl, self, + space.newint(space.BUF_FULL)) if space.isinstance_w(w_result, space.w_buffer): return w_result.writebuf_w(space) raise BufferInterfaceNotFound @@ -232,7 +235,8 @@ def charbuf_w(self, space): w_impl = space.lookup(self, '__buffer__') if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) + w_result = space.get_and_call_function(w_impl, self, + space.newint(space.BUF_FULL_RO)) if space.isinstance_w(w_result, space.w_buffer): return w_result.charbuf_w(space) raise BufferInterfaceNotFound diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py --- a/pypy/module/__builtin__/descriptor.py +++ b/pypy/module/__builtin__/descriptor.py @@ -23,6 +23,14 @@ self.w_objtype = w_type self.w_self = w_obj_or_type + def descr_repr(self, space): + if self.w_objtype is not None: + objtype_name = "<%s object>" % self.w_objtype.getname(space) + else: + objtype_name = 'NULL' + return space.wrap("<super: <class '%s'>, %s>" % ( + self.w_starttype.getname(space), objtype_name)) + def get(self, space, w_obj, w_type=None): if self.w_self is None or space.is_w(w_obj, space.w_None): return self @@ -84,7 +92,10 @@ 'super', __new__ = generic_new_descr(W_Super), __init__ = interp2app(W_Super.descr_init), + __repr__ = interp2app(W_Super.descr_repr), __thisclass__ = interp_attrproperty_w("w_starttype", W_Super), + __self__ = interp_attrproperty_w("w_self", W_Super), + __self_class__ = interp_attrproperty_w("w_objtype", W_Super), __getattribute__ = interp2app(W_Super.getattribute), __get__ = interp2app(W_Super.get), __doc__ = """\ diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py --- a/pypy/module/__builtin__/interp_classobj.py +++ b/pypy/module/__builtin__/interp_classobj.py @@ -38,6 +38,8 @@ class W_ClassObject(W_Root): + _immutable_fields_ = ['bases_w?[*]', 'w_dict?'] + def __init__(self, space, w_name, bases, w_dict): self.name = space.str_w(w_name) make_sure_not_resized(bases) @@ -75,6 +77,7 @@ "__bases__ items must be classes") self.bases_w = bases_w + @jit.unroll_safe def is_subclass_of(self, other): assert isinstance(other, W_ClassObject) if self is other: @@ -313,7 +316,7 @@ # This method ignores the instance dict and the __getattr__. # Returns None if not found. assert isinstance(name, str) - w_value = self.w_class.lookup(space, name) + w_value = jit.promote(self.w_class).lookup(space, name) if w_value is None: return None w_descr_get = space.lookup(w_value, '__get__') diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py --- a/pypy/module/__builtin__/test/test_descriptor.py +++ b/pypy/module/__builtin__/test/test_descriptor.py @@ -250,6 +250,24 @@ assert super(B, B()).__thisclass__ is B assert super(A, B()).__thisclass__ is A + def test_super_self_selfclass(self): + class A(object): + pass + class B(A): + pass + b = B() + assert super(A, b).__self__ is b + assert super(A).__self__ is None + assert super(A, b).__self_class__ is B + assert super(A).__self_class__ is None + + def test_super_repr(self): + class A(object): + def __repr__(self): + return super(A, self).__repr__() + '!' + assert repr(A()).endswith('>!') + assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>" + def test_property_docstring(self): assert property.__doc__.startswith('property') diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py --- a/pypy/module/_sre/__init__.py +++ b/pypy/module/_sre/__init__.py @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): @@ -7,7 +7,7 @@ interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', - 'MAGIC': 'space.wrap(interp_sre.MAGIC)', + 'MAGIC': 'space.newint(20031017)', 'MAXREPEAT': 'space.wrap(interp_sre.MAXREPEAT)', 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -14,7 +14,7 @@ # Constants and exposed functions from rpython.rlib.rsre import rsre_core -from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db +from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db @unwrap_spec(char_ord=int, flags=int) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -358,9 +358,15 @@ elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: if not buflen: - return space.wrap("") - s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen) - return space.wrap(s) + s = "" + else: + # may or may not have a trailing NULL in the buffer. + buf = rffi.cast(rffi.CCHARP, buf) + if buf[buflen - 1] == '\x00': + buflen -= 1 + s = rffi.charp2strn(buf, buflen) + w_s = space.wrap(s) + return space.call_method(w_s, 'decode', space.wrap('mbcs')) elif typ == rwinreg.REG_MULTI_SZ: if not buflen: @@ -460,7 +466,7 @@ return space.newtuple([ convert_from_regdata(space, databuf, length, retType[0]), - space.wrap(retType[0]), + space.wrap(intmask(retType[0])), ]) @unwrap_spec(subkey=str) @@ -612,7 +618,7 @@ space.wrap(rffi.charp2str(valuebuf)), convert_from_regdata(space, databuf, length, retType[0]), - space.wrap(retType[0]), + space.wrap(intmask(retType[0])), ]) @unwrap_spec(index=int) diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -151,6 +151,7 @@ def test_readValues(self): from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey + from _winreg import REG_SZ, REG_EXPAND_SZ key = OpenKey(self.root_key, self.test_key_name) sub_key = OpenKey(key, "sub_key") index = 0 @@ -164,7 +165,10 @@ assert index == len(self.test_data) for name, value, type in self.test_data: - assert QueryValueEx(sub_key, name) == (value, type) + result = QueryValueEx(sub_key, name) + assert result == (value, type) + if type == REG_SZ or type == REG_EXPAND_SZ: + assert isinstance(result[0], unicode) # not string assert EnumKey(key, 0) == "sub_key" raises(EnvironmentError, EnumKey, key, 1) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -120,8 +120,8 @@ constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE -METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O -Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS +METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS +Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES """.split() for name in constant_names: @@ -649,6 +649,7 @@ #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)), ('internal', rffi.VOIDP) )) +Py_bufferP = lltype.Ptr(Py_buffer) @specialize.memo() def is_PyObject(TYPE): @@ -976,8 +977,10 @@ py_type_ready(space, get_capsule_type()) INIT_FUNCTIONS.append(init_types) from pypy.module.posix.interp_posix import add_fork_hook - reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void, - compilation_info=eci) + _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], + lltype.Void, compilation_info=eci) + def reinit_tls(space): + _reinit_tls() add_fork_hook('child', reinit_tls) def init_function(func): diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,13 +1,17 @@ from pypy.interpreter.error import oefmt from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_buffer) + cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER) from pypy.module.cpyext.pyobject import PyObject @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyObject_CheckBuffer(space, w_obj): +def PyObject_CheckBuffer(space, pyobj): """Return 1 if obj supports the buffer interface otherwise 0.""" - return 0 # the bf_getbuffer field is never filled by cpyext + as_buffer = pyobj.c_ob_type.c_tp_as_buffer + flags = pyobj.c_ob_type.c_tp_flags + if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer): + return 1 + return 0 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -123,5 +123,4 @@ pathname = code.co_filename w_mod = importing.add_module(space, w_name) space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname)) - importing.exec_code_module(space, w_mod, code) - return w_mod + return importing.exec_code_module(space, w_mod, code, w_name) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.3.2-alpha0" -#define PYPY_VERSION_NUM 0x05030200 +#define PYPY_VERSION "5.4.1-alpha0" +#define PYPY_VERSION_NUM 0x05040100 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -12,7 +12,7 @@ @cpython_api([PyObject], PyObject) def PyMemoryView_GET_BASE(space, w_obj): # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER - raise NotImplementedError + raise NotImplementedError('PyMemoryView_GET_BUFFER') @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL) def PyMemoryView_GET_BUFFER(space, w_obj): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -3,15 +3,16 @@ import re from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES, - mangle_name, pypy_decl) + mangle_name, pypy_decl, Py_buffer, Py_bufferP) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, - readbufferproc, ssizessizeobjargproc) + readbufferproc, getbufferproc, ssizessizeobjargproc) from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -22,6 +23,9 @@ from rpython.rlib.objectmodel import specialize from rpython.tool.sourcetools import func_renamer from rpython.rtyper.annlowlevel import llhelper +from pypy.module.sys.version import CPYTHON_VERSION + +PY3 = CPYTHON_VERSION[0] == 3 # XXX: Also defined in object.h Py_LT = 0 @@ -298,11 +302,23 @@ # Similar to Py_buffer _immutable_ = True - def __init__(self, ptr, size, w_obj): + def __init__(self, ptr, size, w_obj, format='B', shape=None, + strides=None, ndim=1, itemsize=1, readonly=True): self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.readonly = True + self.format = format + if not shape: + self.shape = [size] + else: + self.shape = shape + if not strides: + self.strides = [1] + else: + self.strides = strides + self.ndim = ndim + self.itemsize = itemsize + self.readonly = readonly def getlength(self): return self.size @@ -313,6 +329,15 @@ def get_raw_address(self): return rffi.cast(rffi.CCHARP, self.ptr) + def getformat(self): + return self.format + + def getshape(self): + return self.shape + + def getitemsize(self): + return self.itemsize + def wrap_getreadbuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: @@ -322,6 +347,30 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.newbuffer(CPyBuffer(ptr[0], size, w_self)) +def wrap_getbuffer(space, w_self, w_args, func): + func_target = rffi.cast(getbufferproc, func) + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if space.len_w(w_args) > 0: + _flags = space.int_w(space.listview(w_args)[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly))) + def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): func_target = rffi.cast(richcmpfunc, func) @@ -486,7 +535,6 @@ def slot_tp_getattro(space, w_self, w_name): return space.call_function(getattr_fn, w_self, w_name) api_func = slot_tp_getattro.api_func - elif name == 'tp_call': call_fn = w_type.getdictvalue(space, '__call__') if call_fn is None: @@ -542,6 +590,21 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(space.get(new_fn, w_self), args) api_func = slot_tp_new.api_func + elif name == 'tp_as_buffer.c_bf_getbuffer': + buff_fn = w_type.getdictvalue(space, '__buffer__') + if buff_fn is None: + return + @cpython_api([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, header=None, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def buff_w(space, w_self, pybuf, flags): + # XXX this is wrong, needs a test + raise oefmt(space.w_NotImplemented, + "calling bf_getbuffer on a builtin type not supported yet") + #args = Arguments(space, [w_self], + # w_stararg=w_args, w_starstararg=w_kwds) + #return space.call_args(space.get(buff_fn, w_self), args) + api_func = buff_w.api_func else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length @@ -850,11 +913,19 @@ slotdefs = eval(slotdefs_str) # PyPy addition slotdefs += ( - TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), + # XXX that might not be what we want! + TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""), ) +if not PY3: + slotdefs += ( + TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), + ) + + # partial sort to solve some slot conflicts: # Number slots before Mapping slots before Sequence slots. +# also prefer the new buffer interface # These are the only conflicts between __name__ methods def slotdef_sort_key(slotdef): if slotdef.slot_name.startswith('tp_as_number'): @@ -863,6 +934,10 @@ return 2 if slotdef.slot_name.startswith('tp_as_sequence'): return 3 + if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer': + return 100 + if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer': + return 101 return 0 slotdefs = sorted(slotdefs, key=slotdef_sort_key) diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/buffer_test.c @@ -0,0 +1,248 @@ +#ifdef _MSC_VER +#define _CRT_SECURE_NO_WARNINGS 1 +#endif +#include <Python.h> +#include <stdlib.h> +#include <stdio.h> + +/* + * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol, + * which is copyright Jake Vanderplas and released under the BSD license + */ + +/* Structure defines a 1-dimensional strided array */ +typedef struct{ + int* arr; + Py_ssize_t length; +} MyArray; + +/* initialize the array with integers 0...length */ +void initialize_MyArray(MyArray* a, long length){ + int i; + a->length = length; + a->arr = (int*)malloc(length * sizeof(int)); + for(i=0; i<length; i++){ + a->arr[i] = i; + } +} + +/* free the memory when finished */ +void deallocate_MyArray(MyArray* a){ + free(a->arr); + a->arr = NULL; +} + +/* tools to print the array */ +char* stringify(MyArray* a, int nmax){ + char* output = (char*) malloc(nmax * 20); + int k, pos = sprintf(&output[0], "["); + + for (k=0; k < a->length && k < nmax; k++){ + pos += sprintf(&output[pos], " %d", a->arr[k]); + } + if(a->length > nmax) + pos += sprintf(&output[pos], "..."); + sprintf(&output[pos], " ]"); + return output; +} + +void print_MyArray(MyArray* a, int nmax){ + char* s = stringify(a, nmax); + printf("%s", s); + free(s); +} + +/* This is where we define the PyMyArray object structure */ +typedef struct { + PyObject_HEAD + /* Type-specific fields go below. */ + MyArray arr; +} PyMyArray; + + +/* This is the __init__ function, implemented in C */ +static int +PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds) +{ + int length = 0; + static char *kwlist[] = {"length", NULL}; + // init may have already been called + if (self->arr.arr != NULL) { + deallocate_MyArray(&self->arr); + } + + if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length)) + return -1; + + if (length < 0) + length = 0; + + initialize_MyArray(&self->arr, length); + + return 0; +} + + +/* this function is called when the object is deallocated */ +static void +PyMyArray_dealloc(PyMyArray* self) +{ + deallocate_MyArray(&self->arr); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* This function returns the string representation of our object */ +static PyObject * +PyMyArray_str(PyMyArray * self) +{ + char* s = stringify(&self->arr, 10); + PyObject* ret = PyUnicode_FromString(s); + free(s); + return ret; +} + +/* Here is the buffer interface function */ +static int +PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags) +{ + PyMyArray* self = (PyMyArray*)obj; + fprintf(stdout, "in PyMyArray_getbuffer\n"); + if (view == NULL) { + fprintf(stdout, "view is NULL\n"); + PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer"); + return -1; + } + if (flags == 0) { + fprintf(stdout, "flags is 0\n"); + PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer"); + return -1; + } + + view->obj = (PyObject*)self; + view->buf = (void*)self->arr.arr; + view->len = self->arr.length * sizeof(int); + view->readonly = 0; + view->itemsize = sizeof(int); + view->format = "i"; // integer + view->ndim = 1; + view->shape = &self->arr.length; // length-1 sequence of dimensions + view->strides = &view->itemsize; // for the simple case we can do this + view->suboffsets = NULL; + view->internal = NULL; + + Py_INCREF(self); // need to increase the reference count + return 0; +} + +static PyBufferProcs PyMyArray_as_buffer = { +#if PY_MAJOR_VERSION < 3 + (readbufferproc)0, + (writebufferproc)0, + (segcountproc)0, + (charbufferproc)0, +#endif + (getbufferproc)PyMyArray_getbuffer, + (releasebufferproc)0, // we do not require any special release function +}; + + +/* Here is the type structure: we put the above functions in the appropriate place + in order to actually define the Python object type */ +static PyTypeObject PyMyArrayType = { + PyVarObject_HEAD_INIT(NULL, 0) + "pymyarray.PyMyArray", /* tp_name */ + sizeof(PyMyArray), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyMyArray_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)PyMyArray_str, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)PyMyArray_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + &PyMyArray_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ + "PyMyArray object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyMyArray_init, /* tp_init */ +}; + +static PyMethodDef buffer_functions[] = { + {NULL, NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "buffer_test", + "Module Doc", + -1, + buffer_functions, + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_buffer_test(void) + +#else + +#define INITERROR return + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +#endif + +PyMODINIT_FUNC +initbuffer_test(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *m= PyModule_Create(&moduledef); +#else + PyObject *m= Py_InitModule("buffer_test", buffer_functions); +#endif + if (m == NULL) + INITERROR; + PyMyArrayType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyMyArrayType) < 0) + INITERROR; + Py_INCREF(&PyMyArrayType); + PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType); +#if PY_MAJOR_VERSION >=3 + return m; +#endif +} diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -87,4 +87,13 @@ module.switch_multiply() res = [1, 2, 3] * arr assert res == [2, 4, 6] + + def test_subclass(self): + module = self.import_module(name='array') + class Sub(module.array): + pass + + arr = Sub('i', [2]) + res = [1, 2, 3] * arr + assert res == [1, 2, 3, 1, 2, 3] diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -92,10 +92,20 @@ link_extra=link_extra, libraries=libraries) from pypy.module.imp.importing import get_so_extension - pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) + ext = get_so_extension(space) + pydname = soname.new(purebasename=modname, ext=ext) soname.rename(pydname) return str(pydname) +def get_so_suffix(): + from imp import get_suffixes, C_EXTENSION + for suffix, mode, typ in get_suffixes(): + if typ == C_EXTENSION: + return suffix + else: + raise RuntimeError("This interpreter does not define a filename " + "suffix for C extensions!") + def compile_extension_module_applevel(space, modname, include_dirs=[], source_files=None, source_strings=None): """ @@ -126,13 +136,9 @@ source_strings=source_strings, compile_extra=compile_extra, link_extra=link_extra) - from imp import get_suffixes, C_EXTENSION - pydname = soname - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - pydname = soname.new(purebasename=modname, ext=suffix) - soname.rename(pydname) - break + ext = get_so_suffix() + pydname = soname.new(purebasename=modname, ext=ext) + soname.rename(pydname) return str(pydname) def freeze_refcnts(self): @@ -145,6 +151,24 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) +class FakeSpace(object): + """Like TinyObjSpace, but different""" + def __init__(self, config): + from distutils.sysconfig import get_python_inc + self.config = config + self.include_dir = get_python_inc() + + def passthrough(self, arg): + return arg + listview = passthrough + str_w = passthrough + + def unwrap(self, args): + try: + return args.str_w(None) + except: + return args + class LeakCheckingTest(object): """Base class for all cpyext tests.""" spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', @@ -433,21 +457,8 @@ self.imported_module_names = [] if self.runappdirect: + fake = FakeSpace(self.space.config) def interp2app(func): - from distutils.sysconfig import get_python_inc - class FakeSpace(object): - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - fake = FakeSpace() - fake.include_dir = get_python_inc() - fake.config = self.space.config def run(*args, **kwargs): for k in kwargs.keys(): if k not in func.unwrap_spec and not k.startswith('w_'): diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -1,17 +1,26 @@ -import pytest from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class TestMemoryViewObject(BaseApiTest): def test_fromobject(self, space, api): - if space.is_true(space.lt(space.sys.get('version_info'), - space.wrap((2, 7)))): - py.test.skip("unsupported before Python 2.7") - w_hello = space.newbytes("hello") + assert api.PyObject_CheckBuffer(w_hello) w_view = api.PyMemoryView_FromObject(w_hello) + w_char = space.call_method(w_view, '__getitem__', space.wrap(0)) + assert space.eq_w(w_char, space.wrap('h')) w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" - @pytest.mark.skipif(True, reason='write a test for this') - def test_get_base_and_get_buffer(self, space, api): - assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER + +class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_buffer_protocol(self): + import struct + module = self.import_module(name='buffer_test') + arr = module.PyMyArray(10) + y = memoryview(arr) + assert y.format == 'i' + assert y.shape == (10,) + s = y[3] + assert len(s) == struct.calcsize('i') + assert s == struct.pack('i', 3) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -17,7 +17,9 @@ generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder, - PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr) + PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr, + Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES, + Py_TPFLAGS_HAVE_INPLACEOPS) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject) @@ -385,6 +387,8 @@ pto.c_tp_basicsize = base_pto.c_tp_basicsize if pto.c_tp_itemsize < base_pto.c_tp_itemsize: pto.c_tp_itemsize = base_pto.c_tp_itemsize + pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES + pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) @@ -608,6 +612,7 @@ bf_getwritebuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER + pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): @@ -774,6 +779,8 @@ pto.c_tp_setattro = base.c_tp_setattro if not pto.c_tp_getattro: pto.c_tp_getattro = base.c_tp_getattro + if not pto.c_tp_as_buffer: + pto.c_tp_as_buffer = base.c_tp_as_buffer finally: Py_DecRef(space, base_pyo) @@ -810,8 +817,13 @@ # inheriting tp_as_* slots base = py_type.c_tp_base if base: - if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number - if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence + if not py_type.c_tp_as_number: + py_type.c_tp_as_number = base.c_tp_as_number + py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES + py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS + if not py_type.c_tp_as_sequence: + py_type.c_tp_as_sequence = base.c_tp_as_sequence + py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py --- a/pypy/module/cpyext/typeobjectdefs.py +++ b/pypy/module/cpyext/typeobjectdefs.py @@ -5,6 +5,7 @@ Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef +from pypy.module.cpyext.api import Py_bufferP P, FT, PyO = Ptr, FuncType, PyObject @@ -58,8 +59,7 @@ writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t)) segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t)) charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t)) -## We don't support new buffer interface for now -getbufferproc = rffi.VOIDP +getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real)) releasebufferproc = rffi.VOIDP diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -597,6 +597,11 @@ @jit.dont_look_inside def load_module(space, w_modulename, find_info, reuse=False): + """Like load_module() in CPython's import.c, this will normally + make a module object, store it in sys.modules, execute code in it, + and then fetch it again from sys.modules. But this logic is not + used if we're calling a PEP302 loader. + """ if find_info is None: return @@ -625,17 +630,15 @@ try: if find_info.modtype == PY_SOURCE: - load_source_module( + return load_source_module( space, w_modulename, w_mod, find_info.filename, find_info.stream.readall(), find_info.stream.try_to_find_file_descriptor()) - return w_mod elif find_info.modtype == PY_COMPILED: magic = _r_long(find_info.stream) timestamp = _r_long(find_info.stream) - load_compiled_module(space, w_modulename, w_mod, find_info.filename, + return load_compiled_module(space, w_modulename, w_mod, find_info.filename, magic, timestamp, find_info.stream.readall()) - return w_mod elif find_info.modtype == PKG_DIRECTORY: w_path = space.newlist([space.wrap(find_info.filename)]) space.setattr(w_mod, space.wrap('__path__'), w_path) @@ -644,14 +647,13 @@ if find_info is None: return w_mod try: - load_module(space, w_modulename, find_info, reuse=True) + w_mod = load_module(space, w_modulename, find_info, + reuse=True) finally: try: find_info.stream.close() except StreamErrors: pass - # fetch the module again, in case of "substitution" - w_mod = check_sys_modules(space, w_modulename) return w_mod elif find_info.modtype == C_EXTENSION and has_so_extension(space): load_c_extension(space, find_info.filename, space.str_w(w_modulename)) @@ -677,13 +679,6 @@ try: if find_info: w_mod = load_module(space, w_modulename, find_info) - try: - w_mod = space.getitem(space.sys.get("modules"), - w_modulename) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
