Author: Armin Rigo <[email protected]>
Branch:
Changeset: r68364:374c311e3d9d
Date: 2013-12-02 21:29 +0000
http://bitbucket.org/pypy/pypy/changeset/374c311e3d9d/
Log: merge heads
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,6 @@
.. branch: less-stringly-ops
Use subclasses of SpaceOperation instead of SpaceOperator objects.
Random cleanups in flowspace and annotator.
+
+.. branch: ndarray-buffer
+adds support for the buffer= argument to the ndarray ctor
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -47,6 +47,9 @@
def get_raw_address(self):
raise ValueError("no raw buffer")
+ def is_writable(self):
+ return False
+
# __________ app-level support __________
def descr_len(self, space):
@@ -135,6 +138,9 @@
__slots__ = () # no extra slot here
+ def is_writable(self):
+ return True
+
def setitem(self, index, char):
"Write a character into the buffer."
raise NotImplementedError # Must be overriden. No bounds checks.
diff --git a/pypy/module/micronumpy/app_numpy.py
b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -12,7 +12,9 @@
if dtype is None:
test = _numpypy.multiarray.array([start, stop, step, 0])
dtype = test.dtype
- arr = _numpypy.multiarray.zeros(int(math.ceil((stop - start) / step)),
dtype=dtype)
+ length = math.ceil((float(stop) - start) / step)
+ length = int(length)
+ arr = _numpypy.multiarray.zeros(length, dtype=dtype)
i = start
for j in range(arr.size):
arr[j] = i
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py
b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -392,6 +392,21 @@
def __del__(self):
free_raw_storage(self.storage, track_allocation=False)
+class ConcreteArrayWithBase(ConcreteArrayNotOwning):
+ def __init__(self, shape, dtype, order, strides, backstrides, storage,
orig_base):
+ ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
+ strides, backstrides, storage)
+ self.orig_base = orig_base
+
+ def base(self):
+ return self.orig_base
+
+
+class ConcreteNonWritableArrayWithBase(ConcreteArrayWithBase):
+ def descr_setitem(self, space, orig_array, w_index, w_value):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "assignment destination is read-only"))
+
class NonWritableArray(ConcreteArray):
def descr_setitem(self, space, orig_array, w_index, w_value):
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -49,11 +49,24 @@
return W_NDimArray(impl)
@staticmethod
- def from_shape_and_storage(space, shape, storage, dtype, order='C',
owning=False, w_subtype=None):
+ def from_shape_and_storage(space, shape, storage, dtype, order='C',
owning=False,
+ w_subtype=None, w_base=None, writable=True):
from pypy.module.micronumpy.arrayimpl import concrete
assert shape
strides, backstrides = calc_strides(shape, dtype, order)
- if owning:
+ if w_base is not None:
+ if owning:
+ raise OperationError(space.w_ValueError,
+ space.wrap("Cannot have owning=True when specifying a
buffer"))
+ if writable:
+ impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
strides,
+ backstrides, storage,
w_base)
+ else:
+ impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype,
order,
+ strides,
backstrides,
+ storage,
w_base)
+
+ elif owning:
# Will free storage when GCd
impl = concrete.ConcreteArray(shape, dtype, order, strides,
backstrides, storage=storage)
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,3 +1,5 @@
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rawstorage import RAW_STORAGE_PTR
from pypy.interpreter.error import operationerrfmt, OperationError
from pypy.interpreter.typedef import TypeDef, GetSetProperty,
make_weakref_descr
from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \
@@ -20,6 +22,7 @@
from rpython.rlib.rstring import StringBuilder
from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation
from pypy.module.micronumpy.conversion_utils import order_converter,
multi_axis_converter
+from pypy.module.micronumpy import support
from pypy.module.micronumpy.constants import *
def _find_shape(space, w_size, dtype):
@@ -1067,13 +1070,35 @@
offset=0, w_strides=None, order='C'):
from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray
from pypy.module.micronumpy.support import calc_strides
- if (offset != 0 or not space.is_none(w_strides) or
- not space.is_none(w_buffer)):
- raise OperationError(space.w_NotImplementedError,
- space.wrap("unsupported param"))
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
shape = _find_shape(space, w_shape, dtype)
+
+ if not space.is_none(w_buffer):
+ if (not space.is_none(w_strides)):
+ raise OperationError(space.w_NotImplementedError,
+ space.wrap("unsupported param"))
+
+ buf = space.buffer_w(w_buffer)
+ try:
+ raw_ptr = buf.get_raw_address()
+ except ValueError:
+ raise OperationError(space.w_TypeError, space.wrap(
+ "Only raw buffers are supported"))
+ if not shape:
+ raise OperationError(space.w_TypeError, space.wrap(
+ "numpy scalars from buffers not supported yet"))
+ totalsize = support.product(shape) * dtype.get_size()
+ if totalsize+offset > buf.getlength():
+ raise OperationError(space.w_TypeError, space.wrap(
+ "buffer is too small for requested array"))
+ storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr)
+ storage = rffi.ptradd(storage, offset)
+ return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+ w_subtype=w_subtype,
+ w_base=w_buffer,
+ writable=buf.is_writable())
+
if not shape:
return W_NDimArray.new_scalar(space, dtype)
if space.is_w(w_subtype, space.gettypefor(W_NDimArray)):
@@ -1093,8 +1118,6 @@
Create an array from an existing buffer, given its address as int.
PyPy-only implementation detail.
"""
- from rpython.rtyper.lltypesystem import rffi
- from rpython.rlib.rawstorage import RAW_STORAGE_PTR
storage = rffi.cast(RAW_STORAGE_PTR, addr)
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype),
diff --git a/pypy/module/micronumpy/test/test_numarray.py
b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -219,6 +219,7 @@
class AppTestNumArray(BaseNumpyAppTest):
spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+
def w_CustomIndexObject(self, index):
class CustomIndexObject(object):
def __init__(self, index):
@@ -381,6 +382,8 @@
assert a.dtype is dtype(int)
a = arange(3, 7, 2)
assert (a == [3, 5]).all()
+ a = arange(3, 8, 2)
+ assert (a == [3, 5, 7]).all()
a = arange(3, dtype=float)
assert (a == [0., 1., 2.]).all()
assert a.dtype is dtype(float)
@@ -2087,6 +2090,69 @@
a = np.ndarray([1], dtype=bool)
assert a[0] == True
+
+class AppTestNumArrayFromBuffer(BaseNumpyAppTest):
+ spaceconfig = dict(usemodules=["micronumpy", "array", "mmap"])
+
+ def setup_class(cls):
+ from rpython.tool.udir import udir
+ BaseNumpyAppTest.setup_class.im_func(cls)
+ cls.w_tmpname = cls.space.wrap(str(udir.join('mmap-')))
+
+ def test_ndarray_from_buffer(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*3)
+ a = np.ndarray((3,), buffer=buf, dtype='i2')
+ a[0] = ord('b')
+ a[1] = ord('a')
+ a[2] = ord('r')
+ assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00']
+ assert a.base is buf
+
+ def test_ndarray_subclass_from_buffer(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*3)
+ class X(np.ndarray):
+ pass
+ a = X((3,), buffer=buf, dtype='i2')
+ assert type(a) is X
+
+ def test_ndarray_from_buffer_and_offset(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*7)
+ buf[0] = 'X'
+ a = np.ndarray((3,), buffer=buf, offset=1, dtype='i2')
+ a[0] = ord('b')
+ a[1] = ord('a')
+ a[2] = ord('r')
+ assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00']
+
+ def test_ndarray_from_buffer_out_of_bounds(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*10) # 20 bytes
+ info = raises(TypeError, "np.ndarray((11,), buffer=buf, dtype='i2')")
+ assert str(info.value).startswith('buffer is too small')
+ info = raises(TypeError, "np.ndarray((5,), buffer=buf, offset=15,
dtype='i2')")
+ assert str(info.value).startswith('buffer is too small')
+
+ def test_ndarray_from_readonly_buffer(self):
+ import numpypy as np
+ from mmap import mmap, ACCESS_READ
+ f = open(self.tmpname, "w+")
+ f.write("hello")
+ f.flush()
+ buf = mmap(f.fileno(), 5, access=ACCESS_READ)
+ a = np.ndarray((5,), buffer=buf, dtype='c')
+ raises(ValueError, "a[0] = 'X'")
+ buf.close()
+ f.close()
+
+
+
class AppTestMultiDim(BaseNumpyAppTest):
def test_init(self):
import numpypy
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -314,6 +314,14 @@
self.check_valid_writeable()
self.mmap.setslice(start, string)
+ def is_writable(self):
+ try:
+ self.mmap.check_writeable()
+ except RMMapError:
+ return False
+ else:
+ return True
+
def get_raw_address(self):
self.check_valid()
return self.mmap.data
diff --git a/pypy/module/pypyjit/interp_jit.py
b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -1,20 +1,20 @@
"""This is not the JIT :-)
-This is transformed to become a JIT by code elsewhere: pypy/jit/*
+This is transformed to become a JIT by code elsewhere: rpython/jit/*
"""
-from rpython.tool.pairtype import extendabletype
from rpython.rlib.rarithmetic import r_uint, intmask
from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside
from rpython.rlib import jit
from rpython.rlib.jit import current_trace_length, unroll_parameters
import pypy.interpreter.pyopcode # for side-effects
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.pycode import PyCode, CO_GENERATOR
+from pypy.interpreter.pycode import CO_GENERATOR
from pypy.interpreter.pyframe import PyFrame
from pypy.interpreter.pyopcode import ExitFrame, Yield
from opcode import opmap
+
PyFrame._virtualizable_ = ['last_instr', 'pycode',
'valuestackdepth', 'locals_stack_w[*]',
'cells[*]',
diff --git a/pypy/testrunner_cfg.py b/pypy/testrunner_cfg.py
--- a/pypy/testrunner_cfg.py
+++ b/pypy/testrunner_cfg.py
@@ -3,7 +3,7 @@
DIRS_SPLIT = [
'translator/c', 'rlib',
- 'rpython/memory', 'jit/metainterp', 'rpython/test',
+ 'memory/test', 'jit/metainterp',
'jit/backend/arm', 'jit/backend/x86',
]
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -14,8 +14,8 @@
eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
-def llexternal(*args):
- return rffi.llexternal(*args, compilation_info=eci)
+def llexternal(*args, **kwargs):
+ return rffi.llexternal(*args, compilation_info=eci, **kwargs)
FILE = lltype.Struct('FILE') # opaque type maybe
@@ -24,9 +24,9 @@
off_t = platform.SimpleType('off_t')
+
CC = platform.configure(CConfig)
OFF_T = CC['off_t']
-
c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
@@ -42,7 +42,8 @@
c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
-c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT, macro=True)
+
c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
rffi.CCHARP)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
