Author: mattip <[email protected]>
Branch:
Changeset: r76225:c5c4df0df240
Date: 2015-03-02 21:05 +0200
http://bitbucket.org/pypy/pypy/changeset/c5c4df0df240/
Log: test, fix for mmap subtype not writeable; asarray not respecting
implementation.start
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -46,7 +46,7 @@
@staticmethod
def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
order='C', owning=False, w_subtype=None,
- w_base=None, writable=True, strides=None):
+ w_base=None, writable=True, strides=None,
start=0):
from pypy.module.micronumpy import concrete
from pypy.module.micronumpy.strides import (calc_strides,
calc_backstrides)
@@ -75,8 +75,9 @@
raise OperationError(space.w_ValueError,
space.wrap("Cannot have owning=True when specifying a
buffer"))
if writable:
- impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
strides,
- backstrides, storage,
w_base)
+ impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
+ strides, backstrides, storage, w_base,
+ start=start)
else:
impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype,
order,
strides,
backstrides,
diff --git a/pypy/module/micronumpy/concrete.py
b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -331,7 +331,7 @@
class ConcreteArrayNotOwning(BaseConcreteArray):
- def __init__(self, shape, dtype, order, strides, backstrides, storage):
+ def __init__(self, shape, dtype, order, strides, backstrides, storage,
start=0):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
@@ -342,6 +342,7 @@
self.strides = strides
self.backstrides = backstrides
self.storage = storage
+ self.start = start
def fill(self, space, box):
self.dtype.itemtype.fill(self.storage, self.dtype.elsize,
@@ -350,7 +351,7 @@
def set_shape(self, space, orig_array, new_shape):
strides, backstrides = calc_strides(new_shape, self.dtype,
self.order)
- return SliceArray(0, strides, backstrides, new_shape, self,
+ return SliceArray(self.start, strides, backstrides, new_shape, self,
orig_array)
def set_dtype(self, space, dtype):
@@ -384,9 +385,10 @@
class ConcreteArrayWithBase(ConcreteArrayNotOwning):
- def __init__(self, shape, dtype, order, strides, backstrides, storage,
orig_base):
+ def __init__(self, shape, dtype, order, strides, backstrides, storage,
+ orig_base, start=0):
ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
- strides, backstrides, storage)
+ strides, backstrides, storage, start)
self.orig_base = orig_base
def base(self):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -99,10 +99,11 @@
for i in range(w_object.get_size()):
elems_w[i] = w_object.implementation.getitem(i * elsize)
else:
+ imp = w_object.implementation
sz = support.product(w_object.get_shape()) * dtype.elsize
return W_NDimArray.from_shape_and_storage(space,
- w_object.get_shape(),w_object.implementation.storage,
- dtype, storage_bytes=sz, w_base=w_object)
+ w_object.get_shape(),imp.storage, dtype, storage_bytes=sz,
+ w_base=w_object, start=imp.start)
else:
# not an array
shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
diff --git a/pypy/module/micronumpy/test/test_subtype.py
b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -2,7 +2,7 @@
class AppTestSupport(BaseNumpyAppTest):
- spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+ spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "mmap"])
def setup_class(cls):
BaseNumpyAppTest.setup_class.im_func(cls)
@@ -476,3 +476,120 @@
a = self.SubType(array([[1, 2], [3, 4]]))
b = array(a, subok=False)
assert type(b) is ndarray
+
+ def test_numpypy_mmap(self):
+ # issue #21 on pypy/numpy
+ from numpy import array, ndarray, arange, dtype as dtypedescr
+ import mmap
+ import os.path
+ from tempfile import mkdtemp
+ import os.path as path
+ valid_filemodes = ["r", "c", "r+", "w+"]
+ writeable_filemodes = ["r+", "w+"]
+ mode_equivalents = {
+ "readonly":"r",
+ "copyonwrite":"c",
+ "readwrite":"r+",
+ "write":"w+"
+ }
+
+ class memmap(ndarray):
+ def __new__(subtype, filename, dtype='uint8', mode='r+', offset=0,
shape=None, order='C'):
+ # Import here to minimize 'import numpy' overhead
+ try:
+ mode = mode_equivalents[mode]
+ except KeyError:
+ if mode not in valid_filemodes:
+ raise ValueError("mode must be one of %s" %
+ (valid_filemodes +
list(mode_equivalents.keys())))
+
+ if hasattr(filename, 'read'):
+ fid = filename
+ own_file = False
+ else:
+ fid = open(filename, (mode == 'c' and 'r' or mode)+'b')
+ own_file = True
+
+ if (mode == 'w+') and shape is None:
+ raise ValueError("shape must be given")
+
+ fid.seek(0, 2)
+ flen = fid.tell()
+ descr = dtypedescr(dtype)
+ _dbytes = descr.itemsize
+
+ if shape is None:
+ bytes = flen - offset
+ if (bytes % _dbytes):
+ fid.close()
+ raise ValueError("Size of available data is not a "
+ "multiple of the data-type size.")
+ size = bytes // _dbytes
+ shape = (size,)
+ else:
+ if not isinstance(shape, tuple):
+ shape = (shape,)
+ size = 1
+ for k in shape:
+ size *= k
+
+ bytes = long(offset + size*_dbytes)
+
+ if mode == 'w+' or (mode == 'r+' and flen < bytes):
+ fid.seek(bytes - 1, 0)
+ fid.write('\0')
+ fid.flush()
+
+ if mode == 'c':
+ acc = mmap.ACCESS_COPY
+ elif mode == 'r':
+ acc = mmap.ACCESS_READ
+ else:
+ acc = mmap.ACCESS_WRITE
+
+ start = offset - offset % mmap.ALLOCATIONGRANULARITY
+ bytes -= start
+ offset -= start
+ mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
+
+ self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm,
+ offset=offset, order=order)
+ self._mmap = mm
+ self.offset = offset
+ self.mode = mode
+
+ if isinstance(filename, basestring):
+ self.filename = os.path.abspath(filename)
+ # py3 returns int for TemporaryFile().name
+ elif (hasattr(filename, "name") and
+ isinstance(filename.name, basestring)):
+ self.filename = os.path.abspath(filename.name)
+ # same as memmap copies (e.g. memmap + 1)
+ else:
+ self.filename = None
+
+ if own_file:
+ fid.close()
+
+ return self
+
+ def flush(self):
+ if self.base is not None and hasattr(self.base, 'flush'):
+ self.base.flush()
+
+ def asarray(obj, itemsize=None, order=None):
+ return array(obj, itemsize, copy=False, order=order)
+
+ filename = path.join(mkdtemp(), 'newfile.dat')
+ data = arange(10*10*36).reshape(10, 10, 36)
+ fp = memmap(filename, dtype='float32', mode='w+', shape=data.shape)
+ vals = [ 242, 507, 255, 505, 315, 316, 308, 506,
+ 309, 255, 211, 505, 315, 316, 308, 506,
+ 309, 255, 255, 711, 194, 232, 711, 711,
+ 709, 710, 709, 710, 882, 897, 711, 245,
+ 711, 711, 168, 245]
+ fp[:] = data
+ fp[5:6][:,4] = vals
+ a = asarray(fp[5:6][:,4])
+ assert (a == vals).all()
+
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -22,6 +22,10 @@
self.check_valid()
return MMapBuffer(self.space, self.mmap, True)
+ def writebuf_w(self, space):
+ self.check_writeable()
+ return MMapBuffer(self.space, self.mmap, False)
+
def close(self):
self.mmap.close()
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
