Author: Richard Plangger <planri...@gmail.com> Branch: ppc-vsx-support Changeset: r86933:0c3342bbecb5 Date: 2016-09-07 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/0c3342bbecb5/
Log: merge default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -30,3 +30,6 @@ 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0 +050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 +050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 +0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.8.1 +Version: 1.8.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.8.1" -__version_info__ = (1, 8, 1) +__version__ = "1.8.2" +__version_info__ = (1, 8, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.8.1" + "\ncompiled with cffi version: 1.8.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.4.1.rst @@ -0,0 +1,64 @@ +========== +PyPy 5.4.1 +========== + +We have released a bugfix for PyPy2.7-v5.4.0, released last week, +due to the following issues: + + * Update list of contributors in documentation and LICENSE file, + this was unfortunately left out of 5.4.0. My apologies to the new + contributors + + * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a + dynamically loadable file + + * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython + + * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors + + * Fix for an issue for translating the sandbox + + * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up + the last byte of a unicode string sometimes + + * Update built-in cffi_ to version 1.8.1 + + * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash + translation with a message asking for help porting it + + * Fix a regression where a PyBytesObject was forced (converted to a RPython + object) when not required, reported as issue #2395 + +Thanks to those who reported the issues. + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _cffi: https://cffi.readthedocs.io +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -7,3 +7,9 @@ .. branch: rpython-resync Backport rpython changes made directly on the py3k and py3.5 branches. + +.. branch: buffer-interface +Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews +in numpypy + + diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1428,6 +1428,9 @@ BUF_FORMAT = 0x0004 BUF_ND = 0x0008 BUF_STRIDES = 0x0010 | BUF_ND + BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES + BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES + BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES BUF_INDIRECT = 0x0100 | BUF_STRIDES BUF_CONTIG_RO = BUF_ND diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.8.1" +VERSION = "1.8.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -105,9 +105,6 @@ return True return False - def _check_only_one_argument_for_union(self, w_ob): - pass - def convert_from_object(self, cdata, w_ob): if not self._copy_from_same(cdata, w_ob): self.convert_struct_from_object(cdata, w_ob, optvarsize=-1) @@ -117,19 +114,24 @@ ) def convert_struct_from_object(self, cdata, w_ob, optvarsize): self.force_lazy_struct() - self._check_only_one_argument_for_union(w_ob) space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): lst_w = space.listview(w_ob) - if len(lst_w) > len(self._fields_list): - raise oefmt(space.w_ValueError, - "too many initializers for '%s' (got %d)", - self.name, len(lst_w)) - for i in range(len(lst_w)): - optvarsize = self._fields_list[i].write_v(cdata, lst_w[i], + j = 0 + for w_obj in lst_w: + try: + while (self._fields_list[j].flags & + W_CField.BF_IGNORE_IN_CTOR): + j += 1 + except IndexError: + raise oefmt(space.w_ValueError, + "too many initializers for '%s' (got %d)", + self.name, len(lst_w)) + optvarsize = self._fields_list[j].write_v(cdata, w_obj, optvarsize) + j += 1 return optvarsize elif space.isinstance_w(w_ob, space.w_dict): @@ -185,14 +187,6 @@ class W_CTypeUnion(W_CTypeStructOrUnion): kind = "union" - def _check_only_one_argument_for_union(self, w_ob): - space = self.space - n = space.int_w(space.len(w_ob)) - if n > 1: - raise oefmt(space.w_ValueError, - "initializer for '%s': %d items given, but only one " - "supported (use a dict if needed)", self.name, n) - class W_CField(W_Root): _immutable_ = True @@ -200,18 +194,21 @@ BS_REGULAR = -1 BS_EMPTY_ARRAY = -2 - def __init__(self, ctype, offset, bitshift, bitsize): + BF_IGNORE_IN_CTOR = 0x01 + + def __init__(self, ctype, offset, bitshift, bitsize, flags): self.ctype = ctype self.offset = offset self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY self.bitsize = bitsize + self.flags = flags # BF_xxx def is_bitfield(self): return self.bitshift >= 0 - def make_shifted(self, offset): + def make_shifted(self, offset, fflags): return W_CField(self.ctype, offset + self.offset, - self.bitshift, self.bitsize) + self.bitshift, self.bitsize, self.flags | fflags) def read(self, cdata): cdata = rffi.ptradd(cdata, self.offset) @@ -341,5 +338,6 @@ offset = interp_attrproperty('offset', W_CField), bitshift = interp_attrproperty('bitshift', W_CField), bitsize = interp_attrproperty('bitsize', W_CField), + flags = interp_attrproperty('flags', W_CField), ) W_CField.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -345,6 +345,11 @@ if alignment < falign and do_align: alignment = falign # + if is_union and i > 0: + fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR + else: + fflags = 0 + # if fbitsize < 0: # not a bitfield: common case @@ -372,7 +377,7 @@ for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: - fld = srcfld.make_shifted(boffset // 8) + fld = srcfld.make_shifted(boffset // 8, fflags) fields_list.append(fld) try: fields_dict[srcfield2names[srcfld]] = fld @@ -382,7 +387,8 @@ w_ctype._custom_field_pos = True else: # a regular field - fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1) + fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1, + fflags) fields_list.append(fld) fields_dict[fname] = fld @@ -489,7 +495,7 @@ bitshift = 8 * ftype.size - fbitsize- bitshift fld = ctypestruct.W_CField(ftype, field_offset_bytes, - bitshift, fbitsize) + bitshift, fbitsize, fflags) fields_list.append(fld) fields_dict[fname] = fld diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.8.1", ("This test_c.py file is for testing a version" +assert __version__ == "1.8.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -2525,6 +2525,25 @@ assert d[2][1].bitshift == -1 assert d[2][1].bitsize == -1 +def test_nested_anonymous_struct_2(): + BInt = new_primitive_type("int") + BStruct = new_struct_type("struct foo") + BInnerUnion = new_union_type("union bar") + complete_struct_or_union(BInnerUnion, [('a1', BInt, -1), + ('a2', BInt, -1)]) + complete_struct_or_union(BStruct, [('b1', BInt, -1), + ('', BInnerUnion, -1), + ('b2', BInt, -1)]) + assert sizeof(BInnerUnion) == sizeof(BInt) + assert sizeof(BStruct) == sizeof(BInt) * 3 + fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields] + assert fields == [ + ('b1', 0 * sizeof(BInt), 0), + ('a1', 1 * sizeof(BInt), 0), + ('a2', 1 * sizeof(BInt), 1), + ('b2', 2 * sizeof(BInt), 0), + ] + def test_sizeof_union(): # a union has the largest alignment of its members, and a total size # that is the largest of its items *possibly further aligned* if diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -450,7 +450,12 @@ # For compatibility assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ finally: - c.shutdown() + try: + c.shutdown() + except _ssl.SSLError: + # If the expected exception was raised, the SSLContext + # can't be shut down yet + pass finally: s.close() diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -597,6 +597,18 @@ def getlength(self): return self.array.len * self.array.itemsize + def getformat(self): + return self.array.typecode + + def getitemsize(self): + return self.array.itemsize + + def getndim(self): + return 1 + + def getstrides(self): + return [self.getitemsize()] + def getitem(self, index): array = self.array data = array._charbuf_start() diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -122,7 +122,7 @@ METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER -Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES +Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS """.split() for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) @@ -645,6 +645,9 @@ ('format', rffi.CCHARP), ('shape', Py_ssize_tP), ('strides', Py_ssize_tP), + ('_format', rffi.UCHAR), + ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)), + ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)), ('suboffsets', Py_ssize_tP), #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)), ('internal', rffi.VOIDP) diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,8 +1,9 @@ from pypy.interpreter.error import oefmt from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER) -from pypy.module.cpyext.pyobject import PyObject + cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP) +from pypy.module.cpyext.pyobject import PyObject, make_ref, incref @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_CheckBuffer(space, pyobj): @@ -33,13 +34,82 @@ raise an error if the object can't support a simpler view of its memory. 0 is returned on success and -1 on error.""" - raise oefmt(space.w_TypeError, - "PyPy does not yet implement the new buffer interface") + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + except ValueError: + raise BufferError("could not create buffer from object") + view.c_len = buf.getlength() + view.c_obj = make_ref(space, w_obj) + ndim = buf.getndim() + view.c_itemsize = buf.getitemsize() + rffi.setintfield(view, 'c_readonly', int(buf.readonly)) + rffi.setintfield(view, 'c_ndim', ndim) + view.c_format = rffi.str2charp(buf.getformat()) + view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw') + view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw') + shape = buf.getshape() + strides = buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] + view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) + view.c_internal = lltype.nullptr(rffi.VOIDP.TO) + return 0 + +def _IsFortranContiguous(view): + ndim = widen(view.c_ndim) + if ndim == 0: + return 1 + if not view.c_strides: + return ndim == 1 + sd = view.c_itemsize + if ndim == 1: + return view.c_shape[0] == 1 or sd == view.c_strides[0] + for i in range(view.c_ndim): + dim = view.c_shape[i] + if dim == 0: + return 1 + if view.c_strides[i] != sd: + return 0 + sd *= dim + return 1 + +def _IsCContiguous(view): + ndim = widen(view.c_ndim) + if ndim == 0: + return 1 + if not view.c_strides: + return ndim == 1 + sd = view.c_itemsize + if ndim == 1: + return view.c_shape[0] == 1 or sd == view.c_strides[0] + for i in range(ndim - 1, -1, -1): + dim = view.c_shape[i] + if dim == 0: + return 1 + if view.c_strides[i] != sd: + return 0 + sd *= dim + return 1 + @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL) -def PyBuffer_IsContiguous(space, view, fortran): +def PyBuffer_IsContiguous(space, view, fort): """Return 1 if the memory defined by the view is C-style (fortran is 'C') or Fortran-style (fortran is 'F') contiguous or either one (fortran is 'A'). Return 0 otherwise.""" - # PyPy only supports contiguous Py_buffers for now. - return 1 + # traverse the strides, checking for consistent stride increases from + # right-to-left (c) or left-to-right (fortran). Copied from cpython + if not view.c_suboffsets: + return 0 + if (fort == 'C'): + return _IsCContiguous(view) + elif (fort == 'F'): + return _IsFortranContiguous(view) + elif (fort == 'A'): + return (_IsCContiguous(view) or _IsFortranContiguous(view)) + return 0 + + diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -14,45 +14,33 @@ ## Implementation of PyBytesObject ## ================================ ## -## The problem -## ----------- +## PyBytesObject has its own ob_sval buffer, so we have two copies of a string; +## one in the PyBytesObject returned from various C-API functions and another +## in the corresponding RPython object. ## -## PyString_AsString() must return a (non-movable) pointer to the underlying -## ob_sval, whereas pypy strings are movable. C code may temporarily store -## this address and use it, as long as it owns a reference to the PyObject. -## There is no "release" function to specify that the pointer is not needed -## any more. +## The following calls can create a PyBytesObject without a correspoinding +## RPython object: ## -## Also, the pointer may be used to fill the initial value of string. This is -## valid only when the string was just allocated, and is not used elsewhere. +## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n) ## -## Solution -## -------- +## In the PyBytesObject returned, the ob_sval buffer may be modified as +## long as the freshly allocated PyBytesObject is not "forced" via a call +## to any of the more sophisticated C-API functions. ## -## PyBytesObject contains two additional members: the ob_size and a pointer to a -## char ob_sval; it may be NULL. -## -## - A string allocated by pypy will be converted into a PyBytesObject with a -## NULL buffer. The first time PyString_AsString() is called, memory is -## allocated (with flavor='raw') and content is copied. -## -## - A string allocated with PyString_FromStringAndSize(NULL, size) will -## allocate a PyBytesObject structure, and a buffer with the specified -## size+1, but the reference won't be stored in the global map; there is no -## corresponding object in pypy. When from_ref() or Py_INCREF() is called, -## the pypy string is created, and added to the global map of tracked -## objects. The buffer is then supposed to be immutable. -## -##- A buffer obtained from PyString_AS_STRING() could be mutable iff -## there is no corresponding pypy object for the string -## -## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a -## similar object. -## -## - PyString_Size() doesn't need to force the object. +## Care has been taken in implementing the functions below, so that +## if they are called with a non-forced PyBytesObject, they will not +## unintentionally force the creation of a RPython object. As long as only these +## are used, the ob_sval buffer is still modifiable: +## +## PyBytes_AsString / PyString_AsString +## PyBytes_AS_STRING / PyString_AS_STRING +## PyBytes_AsStringAndSize / PyString_AsStringAndSize +## PyBytes_Size / PyString_Size +## PyBytes_Resize / PyString_Resize +## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object) ## ## - There could be an (expensive!) check in from_ref() that the buffer still -## corresponds to the pypy gc-managed string. +## corresponds to the pypy gc-managed string, ## PyBytesObjectStruct = lltype.ForwardReference() @@ -156,9 +144,6 @@ "expected string or Unicode object, %T found", from_ref(space, ref)) ref_str = rffi.cast(PyBytesObject, ref) - if not pyobj_has_w_obj(ref): - # XXX Force the ref? - bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) @@ -182,9 +167,6 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) - if not pyobj_has_w_obj(ref): - # force the ref - bytes_realize(space, ref) ref_str = rffi.cast(PyBytesObject, ref) data[0] = ref_str.c_ob_sval if length: diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -142,7 +142,8 @@ typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); -/* Py3k buffer interface */ +/* Py3k buffer interface, adapted for PyPy */ +#define Py_MAX_NDIMS 32 typedef struct bufferinfo { void *buf; PyObject *obj; /* owned reference */ @@ -156,12 +157,14 @@ char *format; Py_ssize_t *shape; Py_ssize_t *strides; - Py_ssize_t *suboffsets; - + Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/ + unsigned char _format; + Py_ssize_t _strides[Py_MAX_NDIMS]; + Py_ssize_t _shape[Py_MAX_NDIMS]; /* static store for shape and strides of mono-dimensional buffers. */ /* Py_ssize_t smalltable[2]; */ - void *internal; + void *internal; /* always NULL for app-level objects */ } Py_buffer; diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,7 +1,8 @@ from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL, - build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject -from rpython.rtyper.lltypesystem import lltype + Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP) +from pypy.module.cpyext.pyobject import PyObject, make_ref, incref +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.objspace.std.memoryobject import W_MemoryView PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview") @@ -12,6 +13,7 @@ @cpython_api([PyObject], PyObject) def PyMemoryView_GET_BASE(space, w_obj): # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER + # XXX needed for numpy on py3k raise NotImplementedError('PyMemoryView_GET_BUFFER') @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL) @@ -20,21 +22,35 @@ object. The object must be a memoryview instance; this macro doesn't check its type, you must do it yourself or you will risk crashes.""" view = lltype.malloc(Py_buffer, flavor='raw', zero=True) - # TODO - fill in fields - ''' - view.c_buf = buf - view.c_len = length - view.c_obj = obj - Py_IncRef(space, obj) - view.c_itemsize = 1 - rffi.setintfield(view, 'c_readonly', readonly) - rffi.setintfield(view, 'c_ndim', 0) - view.c_format = lltype.nullptr(rffi.CCHARP.TO) - view.c_shape = lltype.nullptr(Py_ssize_tP.TO) - view.c_strides = lltype.nullptr(Py_ssize_tP.TO) + if not isinstance(w_obj, W_MemoryView): + return view + ndim = w_obj.buf.getndim() + if ndim >= Py_MAX_NDIMS: + # XXX warn? + return view + try: + view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly) + isstr = False + except ValueError: + w_s = w_obj.descr_tobytes(space) + view.c_obj = make_ref(space, w_s) + rffi.setintfield(view, 'c_readonly', 1) + isstr = True + view.c_len = w_obj.getlength() + view.c_itemsize = w_obj.buf.getitemsize() + rffi.setintfield(view, 'c_ndim', ndim) + view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat()) + view.c_format = rffi.cast(rffi.CCHARP, view.c__format) + view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) + view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) + shape = w_obj.buf.getshape() + strides = w_obj.buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) view.c_internal = lltype.nullptr(rffi.VOIDP.TO) - ''' return view - diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -508,10 +508,9 @@ @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL) def PyBuffer_Release(space, view): """ - Releases a Py_buffer obtained from getbuffer ParseTuple's s*. - - This is not a complete re-implementation of the CPython API; it only - provides a subset of CPython's behavior. + Release the buffer view. This should be called when the buffer is + no longer being used as it may free memory from it """ Py_DecRef(space, view.c_obj) view.c_obj = lltype.nullptr(PyObject.TO) + # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -335,9 +335,15 @@ def getshape(self): return self.shape + def getstrides(self): + return self.strides + def getitemsize(self): return self.itemsize + def getndim(self): + return self.ndim + def wrap_getreadbuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c --- a/pypy/module/cpyext/test/buffer_test.c +++ b/pypy/module/cpyext/test/buffer_test.c @@ -107,14 +107,11 @@ PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags) { PyMyArray* self = (PyMyArray*)obj; - fprintf(stdout, "in PyMyArray_getbuffer\n"); if (view == NULL) { - fprintf(stdout, "view is NULL\n"); PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer"); return -1; } if (flags == 0) { - fprintf(stdout, "flags is 0\n"); PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer"); return -1; } @@ -188,7 +185,131 @@ (initproc)PyMyArray_init, /* tp_init */ }; +static PyObject* +test_buffer(PyObject* self, PyObject* args) +{ + Py_buffer* view = NULL; + PyObject* obj = PyTuple_GetItem(args, 0); + PyObject* memoryview = PyMemoryView_FromObject(obj); + if (memoryview == NULL) + return PyInt_FromLong(-1); + view = PyMemoryView_GET_BUFFER(memoryview); + Py_DECREF(memoryview); + return PyInt_FromLong(view->len); +} + +/* Copied from numpy tests */ +/* + * Create python string from a FLAG and or the corresponding PyBuf flag + * for the use in get_buffer_info. + */ +#define GET_PYBUF_FLAG(FLAG) \ + buf_flag = PyUnicode_FromString(#FLAG); \ + flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ); \ + Py_DECREF(buf_flag); \ + if (flag_matches == 1) { \ + Py_DECREF(tmp); \ + flags |= PyBUF_##FLAG; \ + continue; \ + } \ + else if (flag_matches == -1) { \ + Py_DECREF(tmp); \ + return NULL; \ + } + + +/* + * Get information for a buffer through PyBuf_GetBuffer with the + * corresponding flags or'ed. Note that the python caller has to + * make sure that or'ing those flags actually makes sense. + * More information should probably be returned for future tests. + */ +static PyObject * +get_buffer_info(PyObject *self, PyObject *args) +{ + PyObject *buffer_obj, *pyflags; + PyObject *tmp, *buf_flag; + Py_buffer buffer; + PyObject *shape, *strides; + Py_ssize_t i, n; + int flag_matches; + int flags = 0; + + if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) { + return NULL; + } + + n = PySequence_Length(pyflags); + if (n < 0) { + return NULL; + } + + for (i=0; i < n; i++) { + tmp = PySequence_GetItem(pyflags, i); + if (tmp == NULL) { + return NULL; + } + + GET_PYBUF_FLAG(SIMPLE); + GET_PYBUF_FLAG(WRITABLE); + GET_PYBUF_FLAG(STRIDES); + GET_PYBUF_FLAG(ND); + GET_PYBUF_FLAG(C_CONTIGUOUS); + GET_PYBUF_FLAG(F_CONTIGUOUS); + GET_PYBUF_FLAG(ANY_CONTIGUOUS); + GET_PYBUF_FLAG(INDIRECT); + GET_PYBUF_FLAG(FORMAT); + GET_PYBUF_FLAG(STRIDED); + GET_PYBUF_FLAG(STRIDED_RO); + GET_PYBUF_FLAG(RECORDS); + GET_PYBUF_FLAG(RECORDS_RO); + GET_PYBUF_FLAG(FULL); + GET_PYBUF_FLAG(FULL_RO); + GET_PYBUF_FLAG(CONTIG); + GET_PYBUF_FLAG(CONTIG_RO); + + Py_DECREF(tmp); + + /* One of the flags must match */ + PyErr_SetString(PyExc_ValueError, "invalid flag used."); + return NULL; + } + + if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) { + return NULL; + } + + if (buffer.shape == NULL) { + Py_INCREF(Py_None); + shape = Py_None; + } + else { + shape = PyTuple_New(buffer.ndim); + for (i=0; i < buffer.ndim; i++) { + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i])); + } + } + + if (buffer.strides == NULL) { + Py_INCREF(Py_None); + strides = Py_None; + } + else { + strides = PyTuple_New(buffer.ndim); + for (i=0; i < buffer.ndim; i++) { + PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i])); + } + } + + PyBuffer_Release(&buffer); + return Py_BuildValue("(NN)", shape, strides); +} + + + static PyMethodDef buffer_functions[] = { + {"test_buffer", (PyCFunction)test_buffer, METH_VARARGS, NULL}, + {"get_buffer_info", (PyCFunction)get_buffer_info, METH_VARARGS, NULL}, {NULL, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -183,8 +183,27 @@ Py_INCREF(Py_None); return Py_None; """), + ("c_only", "METH_NOARGS", + """ + int ret; + char * buf2; + PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024); + if (!obj) + return NULL; + buf2 = PyBytes_AsString(obj); + if (!buf2) + return NULL; + /* buf should not have been forced, issue #2395 */ + ret = _PyBytes_Resize(&obj, 512); + if (ret < 0) + return NULL; + Py_DECREF(obj); + Py_INCREF(Py_None); + return Py_None; + """), ]) module.getbytes() + module.c_only() def test_py_string_as_string_Unicode(self): module = self.import_extension('foo', [ diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -1,6 +1,6 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - +from rpython.rlib.buffer import StringBuffer class TestMemoryViewObject(BaseApiTest): def test_fromobject(self, space, api): @@ -12,6 +12,12 @@ w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" + def test_frombuffer(self, space, api): + w_buf = space.newbuffer(StringBuffer("hello")) + w_memoryview = api.PyMemoryView_FromObject(w_buf) + w_view = api.PyMemoryView_GET_BUFFER(w_memoryview) + ndim = w_view.c_ndim + assert ndim == 1 class AppTestBufferProtocol(AppTestCpythonExtensionBase): def test_buffer_protocol(self): @@ -21,6 +27,25 @@ y = memoryview(arr) assert y.format == 'i' assert y.shape == (10,) + assert len(y) == 10 s = y[3] assert len(s) == struct.calcsize('i') assert s == struct.pack('i', 3) + viewlen = module.test_buffer(arr) + assert viewlen == y.itemsize * len(y) + + def test_buffer_info(self): + from _numpypy import multiarray as np + module = self.import_module(name='buffer_test') + get_buffer_info = module.get_buffer_info + # test_export_flags from numpy test_multiarray + raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) + # test_relaxed_strides from numpy test_multiarray + arr = np.zeros((1, 10)) + if arr.flags.f_contiguous: + shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS']) + assert strides[0] == 8 + arr = np.ones((10, 1), order='F') + shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) + assert strides[-1] == 8 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -293,6 +293,8 @@ STRUCT_TYPE = PyNumberMethods elif slot_names[0] == 'c_tp_as_sequence': STRUCT_TYPE = PySequenceMethods + elif slot_names[0] == 'c_tp_as_buffer': + STRUCT_TYPE = PyBufferProcs else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -259,10 +259,8 @@ raise oefmt(space.w_ValueError, "Empty module name") w = space.wrap - if w_fromlist is not None and space.is_true(w_fromlist): - fromlist_w = space.fixedview(w_fromlist) - else: - fromlist_w = None + if w_fromlist is not None and not space.is_true(w_fromlist): + w_fromlist = None rel_modulename = None if (level != 0 and w_globals is not None and @@ -284,19 +282,19 @@ w_mod = None else: w_mod = absolute_import(space, rel_modulename, rel_level, - fromlist_w, tentative=True) + w_fromlist, tentative=True) else: w_mod = absolute_import(space, rel_modulename, rel_level, - fromlist_w, tentative=False) + w_fromlist, tentative=False) if w_mod is not None: return w_mod - w_mod = absolute_import(space, modulename, 0, fromlist_w, tentative=0) + w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0) if rel_modulename is not None: space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None) return w_mod -def absolute_import(space, modulename, baselevel, fromlist_w, tentative): +def absolute_import(space, modulename, baselevel, w_fromlist, tentative): # Short path: check in sys.modules, but only if there is no conflict # on the import lock. In the situation of 'import' statements # inside tight loops, this should be true, and absolute_import_try() @@ -304,25 +302,25 @@ # if the import lock is currently held by another thread, then we # have to wait, and so shouldn't use the fast path. if not getimportlock(space).lock_held_by_someone_else(): - w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w) + w_mod = absolute_import_try(space, modulename, baselevel, w_fromlist) if w_mod is not None and not space.is_w(w_mod, space.w_None): return w_mod return absolute_import_with_lock(space, modulename, baselevel, - fromlist_w, tentative) + w_fromlist, tentative) @jit.dont_look_inside def absolute_import_with_lock(space, modulename, baselevel, - fromlist_w, tentative): + w_fromlist, tentative): lock = getimportlock(space) lock.acquire_lock() try: return _absolute_import(space, modulename, baselevel, - fromlist_w, tentative) + w_fromlist, tentative) finally: lock.release_lock(silent_after_fork=True) @jit.unroll_safe -def absolute_import_try(space, modulename, baselevel, fromlist_w): +def absolute_import_try(space, modulename, baselevel, w_fromlist): """ Only look up sys.modules, not actually try to load anything """ w_path = None @@ -330,7 +328,7 @@ if '.' not in modulename: w_mod = check_sys_modules_w(space, modulename) first = w_mod - if fromlist_w is not None and w_mod is not None: + if w_fromlist is not None and w_mod is not None: w_path = try_getattr(space, w_mod, space.wrap('__path__')) else: level = 0 @@ -345,28 +343,36 @@ return None if level == baselevel: first = w_mod - if fromlist_w is not None: + if w_fromlist is not None: w_path = try_getattr(space, w_mod, space.wrap('__path__')) level += 1 - if fromlist_w is not None: + if w_fromlist is not None: + # bit artificial code but important to not just unwrap w_fromlist + # to get a better trace. if it is unwrapped, the immutability of the + # tuple is lost if w_path is not None: - if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0], - space.wrap('*')): + length = space.len_w(w_fromlist) + if length == 1 and space.eq_w( + space.getitem(w_fromlist, space.wrap(0)), + space.wrap('*')): w_all = try_getattr(space, w_mod, space.wrap('__all__')) if w_all is not None: - fromlist_w = space.fixedview(w_all) + w_fromlist = w_all else: - fromlist_w = [] + w_fromlist = None # "from x import *" with x already imported and no x.__all__ # always succeeds without doing more imports. It will # just copy everything from x.__dict__ as it is now. - for w_name in fromlist_w: - if try_getattr(space, w_mod, w_name) is None: - return None + + if w_fromlist is not None: + for i in range(length): + w_name = space.getitem(w_fromlist, space.wrap(i)) + if try_getattr(space, w_mod, w_name) is None: + return None return w_mod return first -def _absolute_import(space, modulename, baselevel, fromlist_w, tentative): +def _absolute_import(space, modulename, baselevel, w_fromlist, tentative): w = space.wrap if '/' in modulename or '\\' in modulename: @@ -394,18 +400,23 @@ w_path = try_getattr(space, w_mod, w('__path__')) level += 1 - if fromlist_w is not None: + if w_fromlist is not None: if w_path is not None: - if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],w('*')): + length = space.len_w(w_fromlist) + if length == 1 and space.eq_w( + space.getitem(w_fromlist, space.wrap(0)), + space.wrap('*')): w_all = try_getattr(space, w_mod, w('__all__')) if w_all is not None: - fromlist_w = space.fixedview(w_all) + w_fromlist = w_all else: - fromlist_w = [] - for w_name in fromlist_w: - if try_getattr(space, w_mod, w_name) is None: - load_part(space, w_path, prefix, space.str0_w(w_name), - w_mod, tentative=1) + w_fromlist = None + if w_fromlist is not None: + for i in range(length): + w_name = space.getitem(w_fromlist, space.wrap(i)) + if try_getattr(space, w_mod, w_name) is None: + load_part(space, w_path, prefix, space.str0_w(w_name), + w_mod, tentative=1) return w_mod else: return first diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -460,6 +460,9 @@ def getdictvalue(self, space, key): return self.items[key] + def descr_memoryview(self, space, buf): + raise oefmt(space.w_TypeError, "error") + class IterDictObject(W_Root): def __init__(self, space, w_dict): self.space = space diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -377,7 +377,25 @@ def __exit__(self, typ, value, traceback): keepalive_until_here(self) - def get_buffer(self, space, readonly): + def get_buffer(self, space, flags): + errtype = space.w_ValueError # should be BufferError, numpy does this instead + if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and + not self.flags & NPY.ARRAY_C_CONTIGUOUS): + raise oefmt(errtype, "ndarray is not C-contiguous") + if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and + not self.flags & NPY.ARRAY_F_CONTIGUOUS): + raise oefmt(errtype, "ndarray is not Fortran contiguous") + if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and + not (self.flags & NPY.ARRAY_F_CONTIGUOUS and + self.flags & NPY.ARRAY_C_CONTIGUOUS)): + raise oefmt(errtype, "ndarray is not contiguous") + if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and + not self.flags & NPY.ARRAY_C_CONTIGUOUS): + raise oefmt(errtype, "ndarray is not C-contiguous") + if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and + not self.flags & NPY.ARRAY_WRITEABLE): + raise oefmt(errtype, "buffer source array is read-only") + readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE return ArrayBuffer(self, readonly) def astype(self, space, dtype, order, copy=True): @@ -695,6 +713,7 @@ index + self.impl.start) def setitem(self, index, v): + # XXX what if self.readonly? raw_storage_setitem(self.impl.storage, index + self.impl.start, rffi.cast(lltype.Char, v)) diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -1,4 +1,5 @@ from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.baseobjspace import BufferInterfaceNotFound from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from rpython.rlib.buffer import SubBuffer from rpython.rlib.rstring import strip_spaces @@ -42,7 +43,7 @@ raise oefmt(space.w_ValueError, "object __array__ method not producing an array") -def try_interface_method(space, w_object): +def try_interface_method(space, w_object, copy): try: w_interface = space.getattr(w_object, space.wrap("__array_interface__")) if w_interface is None: @@ -81,17 +82,20 @@ raise oefmt(space.w_ValueError, "__array_interface__ could not decode dtype %R", w_dtype ) - if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)): + if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or + space.isinstance_w(w_data, space.w_list)): data_w = space.listview(w_data) - data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0])) - read_only = True # XXX why not space.is_true(data_w[1]) + w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0])) + read_only = space.is_true(data_w[1]) or copy offset = 0 - return W_NDimArray.from_shape_and_storage(space, shape, data, - dtype, strides=strides, start=offset), read_only + w_base = w_object + if read_only: + w_base = None + return W_NDimArray.from_shape_and_storage(space, shape, w_data, + dtype, w_base=w_base, strides=strides, + start=offset), read_only if w_data is None: - data = w_object - else: - data = w_data + w_data = w_object w_offset = space.finditem(w_interface, space.wrap('offset')) if w_offset is None: offset = 0 @@ -101,7 +105,7 @@ if strides is not None: raise oefmt(space.w_NotImplementedError, "__array_interface__ strides not fully supported yet") - arr = frombuffer(space, data, dtype, support.product(shape), offset) + arr = frombuffer(space, w_data, dtype, support.product(shape), offset) new_impl = arr.implementation.reshape(arr, shape) return W_NDimArray(new_impl), False @@ -110,6 +114,78 @@ return None, False raise +def _descriptor_from_pep3118_format(space, c_format): + descr = descriptor.decode_w_dtype(space, space.wrap(c_format)) + if descr: + return descr + msg = "invalid PEP 3118 format string: '%s'" % c_format + space.warn(space.wrap(msg), space.w_RuntimeWarning) + return None + +def _array_from_buffer_3118(space, w_object, dtype): + try: + w_buf = space.call_method(space.builtin, "memoryview", w_object) + except OperationError as e: + if e.match(space, space.w_TypeError): + # object does not have buffer interface + return w_object + raise + format = space.getattr(w_buf,space.newbytes('format')) + if format: + descr = _descriptor_from_pep3118_format(space, space.str_w(format)) + if not descr: + return w_object + if dtype and descr: + raise oefmt(space.w_NotImplementedError, + "creating an array from a memoryview while specifying dtype " + "not supported") + if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): + msg = ("Item size computed from the PEP 3118 buffer format " + "string does not match the actual item size.") + space.warn(space.wrap(msg), space.w_RuntimeWarning) + return w_object + dtype = descr + elif not dtype: + dtype = descriptor.get_dtype_cache(space).w_stringdtype + dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))) + nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim'))) + shape = [space.int_w(d) for d in space.listview( + space.getattr(w_buf, space.newbytes('shape')))] + strides = [] + buflen = space.len_w(w_buf) * dtype.elsize + if shape: + strides = [space.int_w(d) for d in space.listview( + space.getattr(w_buf, space.newbytes('strides')))] + if not strides: + d = buflen + strides = [0] * nd + for k in range(nd): + if shape[k] > 0: + d /= shape[k] + strides[k] = d + else: + if nd == 1: + shape = [buflen / dtype.elsize, ] + strides = [dtype.elsize, ] + elif nd > 1: + msg = ("ndim computed from the PEP 3118 buffer format " + "is greater than 1, but shape is NULL.") + space.warn(space.wrap(msg), space.w_RuntimeWarning) + return w_object + try: + w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address'))) + except OperationError as e: + if e.match(space, space.w_ValueError): + return w_object + else: + raise e + writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly'))) + w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data, + storage_bytes=buflen, dtype=dtype, w_base=w_object, + writable=writable, strides=strides) + if w_ret: + return w_ret + return w_object @unwrap_spec(ndmin=int, copy=bool, subok=bool) def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False, @@ -127,6 +203,7 @@ def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False): + from pypy.module.micronumpy.boxes import W_GenericBox # numpy testing calls array(type(array([]))) and expects a ValueError if space.isinstance_w(w_object, space.w_type): raise oefmt(space.w_ValueError, "cannot create ndarray from type instance") @@ -134,13 +211,19 @@ dtype = descriptor.decode_w_dtype(space, w_dtype) if not isinstance(w_object, W_NDimArray): w_array = try_array_method(space, w_object, w_dtype) - if w_array is not None: + if w_array is None: + if ( not space.isinstance_w(w_object, space.w_str) and + not space.isinstance_w(w_object, space.w_unicode) and + not isinstance(w_object, W_GenericBox)): + # use buffer interface + w_object = _array_from_buffer_3118(space, w_object, dtype) + else: # continue with w_array, but do further operations in place w_object = w_array copy = False dtype = w_object.get_dtype() if not isinstance(w_object, W_NDimArray): - w_array, _copy = try_interface_method(space, w_object) + w_array, _copy = try_interface_method(space, w_object, copy) if w_array is not None: w_object = w_array copy = _copy diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -805,19 +805,19 @@ return w_result def buffer_w(self, space, flags): - return self.implementation.get_buffer(space, True) + return self.implementation.get_buffer(space, flags) def readbuf_w(self, space): - return self.implementation.get_buffer(space, True) + return self.implementation.get_buffer(space, space.BUF_FULL_RO) def writebuf_w(self, space): - return self.implementation.get_buffer(space, False) + return self.implementation.get_buffer(space, space.BUF_FULL) def charbuf_w(self, space): - return self.implementation.get_buffer(space, True).as_str() + return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str() def descr_get_data(self, space): - return space.newbuffer(self.implementation.get_buffer(space, False)) + return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL)) @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -3215,7 +3215,9 @@ raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)})) a = array([1, 2, 3]) - b = array(Dummy(a.__array_interface__)) + d = Dummy(a.__array_interface__) + b = array(d) + assert b.base is None b[1] = 200 assert a[1] == 2 # upstream compatibility, is this a bug? interface_a = a.__array_interface__ @@ -3226,6 +3228,8 @@ interface_b.pop('data') interface_a.pop('data') assert interface_a == interface_b + b = array(d, copy=False) + assert b.base is d b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1', 'data': 'a'*100})) @@ -3594,6 +3598,7 @@ cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2)) cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4)) cls.w_ulongval = cls.space.wrap(struct.pack('L', 12)) + cls.w_one = cls.space.wrap(struct.pack('i', 1)) def test_frombuffer(self): import numpy as np @@ -3645,8 +3650,6 @@ else: EMPTY = None x = np.array([1, 2, 3, 4, 5], dtype='i') - y = memoryview('abc') - assert y.format == 'B' y = memoryview(x) assert y.format == 'i' assert y.shape == (5,) @@ -3654,6 +3657,16 @@ assert y.strides == (4,) assert y.suboffsets == EMPTY assert y.itemsize == 4 + assert isinstance(y, memoryview) + assert y[0] == self.one + assert (np.array(y) == x).all() + + x = np.array([0, 0, 0, 0], dtype='O') + y = memoryview(x) + # handles conversion of address to pinned object? + z = np.array(y) + assert z.dtype == 'O' + assert (z == x).all() def test_fromstring(self): import sys diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -702,3 +702,32 @@ ret = obj.sum() print type(ret) assert ret.info == 'spam' + + def test_ndarray_subclass_assigns_base(self): + import numpy as np + init_called = [] + class _DummyArray(object): + """ Dummy object that just exists to hang __array_interface__ dictionaries + and possibly keep alive a reference to a base array. + """ + def __init__(self, interface, base=None): + self.__array_interface__ = interface + init_called.append(1) + self.base = base + + x = np.zeros(10) + d = _DummyArray(x.__array_interface__, base=x) + y = np.array(d, copy=False) + assert sum(init_called) == 1 + assert y.base is d + + x = np.zeros((0,), dtype='float32') + intf = x.__array_interface__.copy() + intf["strides"] = x.strides + x.__array_interface__["strides"] = x.strides + d = _DummyArray(x.__array_interface__, base=x) + y = np.array(d, copy=False) + assert sum(init_called) == 2 + assert y.base is d + + diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1851,7 +1851,7 @@ arr.gcstruct) def read(self, arr, i, offset, dtype): - if arr.gcstruct is V_OBJECTSTORE: + if arr.gcstruct is V_OBJECTSTORE and not arr.base(): raise oefmt(self.space.w_NotImplementedError, "cannot read object from array with no gc hook") return self.box(self._read(arr.storage, i, offset)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_import.py b/pypy/module/pypyjit/test_pypy_c/test_import.py --- a/pypy/module/pypyjit/test_pypy_c/test_import.py +++ b/pypy/module/pypyjit/test_pypy_c/test_import.py @@ -38,3 +38,27 @@ # call_may_force(absolute_import_with_lock). for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")): assert 'call' not in opname # no call-like opcode + + def test_import_fast_path(self, tmpdir): + print tmpdir + pkg = tmpdir.join('mypkg').ensure(dir=True) + subdir = pkg.join("sub").ensure(dir=True) + pkg.join('__init__.py').write("") + subdir.join('__init__.py').write("") + subdir.join('mod.py').write(str(py.code.Source(""" + def do_the_import(): + import sys + """))) + def main(path, n): + def do_the_import(): + from mypkg.sub import mod + import sys + sys.path.append(path) + for i in range(n): + do_the_import() + # + log = self.run(main, [str(tmpdir), 300]) + loop, = log.loops_by_filename(self.filepath) + # check that no string compares and other calls are there + for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")): + assert 'call' not in opname # no call-like opcode diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1415,6 +1415,7 @@ assert p.b == 12 assert p.c == 14 assert p.d == 14 + py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0]) def test_nested_field_offset_align(self): ffi = FFI(backend=self.Backend()) @@ -1454,14 +1455,42 @@ assert p.b == 0 assert p.c == 14 assert p.d == 14 - p = ffi.new("union foo_u *", {'b': 12}) - assert p.a == 0 + p = ffi.new("union foo_u *", {'a': -63, 'b': 12}) + assert p.a == -63 assert p.b == 12 - assert p.c == 0 - assert p.d == 0 - # we cannot specify several items in the dict, even though - # in theory in this particular case it would make sense - # to give both 'a' and 'b' + assert p.c == -63 + assert p.d == -63 + p = ffi.new("union foo_u *", [123, 456]) + assert p.a == 123 + assert p.b == 456 + assert p.c == 123 + assert p.d == 123 + py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0]) + + def test_nested_anonymous_struct_2(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { + int a; + union { int b; union { int c, d; }; }; + int e; + }; + """) + assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT + p = ffi.new("struct foo_s *", [11, 22, 33]) + assert p.a == 11 + assert p.b == p.c == p.d == 22 + assert p.e == 33 + py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44]) + FOO = ffi.typeof("struct foo_s") + fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields] + assert fields == [ + ('a', 0 * SIZE_OF_INT, 0), + ('b', 1 * SIZE_OF_INT, 0), + ('c', 1 * SIZE_OF_INT, 1), + ('d', 1 * SIZE_OF_INT, 1), + ('e', 2 * SIZE_OF_INT, 0), + ] def test_cast_to_array_type(self): ffi = FFI(backend=self.Backend()) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py @@ -35,6 +35,9 @@ def test_nested_anonymous_union(self): py.test.skip("ctypes backend: not supported: nested anonymous union") + def test_nested_anonymous_struct_2(self): + py.test.skip("ctypes backend: not supported: nested anonymous union") + def test_CData_CType_2(self): if sys.version_info >= (3,): py.test.skip("ctypes backend: not supported in Python 3: CType") diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -64,6 +64,8 @@ def setitem_str(self, w_dict, key, w_value): cell = self.getdictvalue_no_unwrapping(w_dict, key) + #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell: + # print "WARNING", key, w_value, cell, self return self._setitem_str_cell_known(cell, w_dict, key, w_value) def _setitem_str_cell_known(self, cell, w_dict, key, w_value): diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -14,6 +14,7 @@ """Implement the built-in 'memoryview' type as a wrapper around an interp-level buffer. """ + _attrs_ = ['buf'] def __init__(self, buf): assert isinstance(buf, Buffer) @@ -115,7 +116,7 @@ self.buf.setslice(start, value.as_str()) def descr_len(self, space): - return space.wrap(self.buf.getlength()) + return space.wrap(self.buf.getlength() / self.buf.getitemsize()) def w_get_format(self, space): return space.wrap(self.buf.getformat()) diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,7 +1,7 @@ # Edit these appropriately before running this script maj=5 min=4 -rev=0 +rev=1 branchname=release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy2.7-v$maj.$min.$rev # ==OR== release-$maj.$min _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
