Author: Antonio Cuni <anto.c...@gmail.com> Branch: Changeset: r68359:71a0ee32f321 Date: 2013-12-02 17:49 +0100 http://bitbucket.org/pypy/pypy/changeset/71a0ee32f321/
Log: merge the ndarray-buffer branch, which adds support for the buffer= argument to the ndarray ctor. This is needed e.g. to use numpy.memmap() diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -47,6 +47,9 @@ def get_raw_address(self): raise ValueError("no raw buffer") + def is_writable(self): + return False + # __________ app-level support __________ def descr_len(self, space): @@ -135,6 +138,9 @@ __slots__ = () # no extra slot here + def is_writable(self): + return True + def setitem(self, index, char): "Write a character into the buffer." raise NotImplementedError # Must be overriden. No bounds checks. diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -392,6 +392,21 @@ def __del__(self): free_raw_storage(self.storage, track_allocation=False) +class ConcreteArrayWithBase(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base): + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, + strides, backstrides, storage) + self.orig_base = orig_base + + def base(self): + return self.orig_base + + +class ConcreteNonWritableArrayWithBase(ConcreteArrayWithBase): + def descr_setitem(self, space, orig_array, w_index, w_value): + raise OperationError(space.w_ValueError, space.wrap( + "assignment destination is read-only")) + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, orig_array, w_index, w_value): diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -49,11 +49,24 @@ return W_NDimArray(impl) @staticmethod - def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, w_subtype=None): + def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, + w_subtype=None, w_base=None, writable=True): from pypy.module.micronumpy.arrayimpl import concrete assert shape strides, backstrides = calc_strides(shape, dtype, order) - if owning: + if w_base is not None: + if owning: + raise OperationError(space.w_ValueError, + space.wrap("Cannot have owning=True when specifying a buffer")) + if writable: + impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides, + backstrides, storage, w_base) + else: + impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order, + strides, backstrides, + storage, w_base) + + elif owning: # Will free storage when GCd impl = concrete.ConcreteArray(shape, dtype, order, strides, backstrides, storage=storage) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,3 +1,5 @@ +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rawstorage import RAW_STORAGE_PTR from pypy.interpreter.error import operationerrfmt, OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \ @@ -20,6 +22,7 @@ from rpython.rlib.rstring import StringBuilder from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation from pypy.module.micronumpy.conversion_utils import order_converter, multi_axis_converter +from pypy.module.micronumpy import support from pypy.module.micronumpy.constants import * def _find_shape(space, w_size, dtype): @@ -1067,13 +1070,35 @@ offset=0, w_strides=None, order='C'): from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray from pypy.module.micronumpy.support import calc_strides - if (offset != 0 or not space.is_none(w_strides) or - not space.is_none(w_buffer)): - raise OperationError(space.w_NotImplementedError, - space.wrap("unsupported param")) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) shape = _find_shape(space, w_shape, dtype) + + if not space.is_none(w_buffer): + if (not space.is_none(w_strides)): + raise OperationError(space.w_NotImplementedError, + space.wrap("unsupported param")) + + buf = space.buffer_w(w_buffer) + try: + raw_ptr = buf.get_raw_address() + except ValueError: + raise OperationError(space.w_TypeError, space.wrap( + "Only raw buffers are supported")) + if not shape: + raise OperationError(space.w_TypeError, space.wrap( + "numpy scalars from buffers not supported yet")) + totalsize = support.product(shape) * dtype.get_size() + if totalsize+offset > buf.getlength(): + raise OperationError(space.w_TypeError, space.wrap( + "buffer is too small for requested array")) + storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) + storage = rffi.ptradd(storage, offset) + return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, + w_subtype=w_subtype, + w_base=w_buffer, + writable=buf.is_writable()) + if not shape: return W_NDimArray.new_scalar(space, dtype) if space.is_w(w_subtype, space.gettypefor(W_NDimArray)): @@ -1093,8 +1118,6 @@ Create an array from an existing buffer, given its address as int. PyPy-only implementation detail. """ - from rpython.rtyper.lltypesystem import rffi - from rpython.rlib.rawstorage import RAW_STORAGE_PTR storage = rffi.cast(RAW_STORAGE_PTR, addr) dtype = space.interp_w(interp_dtype.W_Dtype, space.call_function(space.gettypefor(interp_dtype.W_Dtype), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -219,6 +219,7 @@ class AppTestNumArray(BaseNumpyAppTest): spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + def w_CustomIndexObject(self, index): class CustomIndexObject(object): def __init__(self, index): @@ -2089,6 +2090,69 @@ a = np.ndarray([1], dtype=bool) assert a[0] == True + +class AppTestNumArrayFromBuffer(BaseNumpyAppTest): + spaceconfig = dict(usemodules=["micronumpy", "array", "mmap"]) + + def setup_class(cls): + from rpython.tool.udir import udir + BaseNumpyAppTest.setup_class.im_func(cls) + cls.w_tmpname = cls.space.wrap(str(udir.join('mmap-'))) + + def test_ndarray_from_buffer(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*3) + a = np.ndarray((3,), buffer=buf, dtype='i2') + a[0] = ord('b') + a[1] = ord('a') + a[2] = ord('r') + assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00'] + assert a.base is buf + + def test_ndarray_subclass_from_buffer(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*3) + class X(np.ndarray): + pass + a = X((3,), buffer=buf, dtype='i2') + assert type(a) is X + + def test_ndarray_from_buffer_and_offset(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*7) + buf[0] = 'X' + a = np.ndarray((3,), buffer=buf, offset=1, dtype='i2') + a[0] = ord('b') + a[1] = ord('a') + a[2] = ord('r') + assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00'] + + def test_ndarray_from_buffer_out_of_bounds(self): + import numpypy as np + import array + buf = array.array('c', ['\x00']*2*10) # 20 bytes + info = raises(TypeError, "np.ndarray((11,), buffer=buf, dtype='i2')") + assert str(info.value).startswith('buffer is too small') + info = raises(TypeError, "np.ndarray((5,), buffer=buf, offset=15, dtype='i2')") + assert str(info.value).startswith('buffer is too small') + + def test_ndarray_from_readonly_buffer(self): + import numpypy as np + from mmap import mmap, ACCESS_READ + f = open(self.tmpname, "w+") + f.write("hello") + f.flush() + buf = mmap(f.fileno(), 5, access=ACCESS_READ) + a = np.ndarray((5,), buffer=buf, dtype='c') + raises(ValueError, "a[0] = 'X'") + buf.close() + f.close() + + + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import numpypy diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -314,6 +314,14 @@ self.check_valid_writeable() self.mmap.setslice(start, string) + def is_writable(self): + try: + self.mmap.check_writeable() + except RMMapError: + return False + else: + return True + def get_raw_address(self): self.check_valid() return self.mmap.data _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit