Author: Armin Rigo <[email protected]>
Branch:
Changeset: r80605:04570c9524ed
Date: 2015-11-09 08:53 +0100
http://bitbucket.org/pypy/pypy/changeset/04570c9524ed/
Log: merge heads
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -18,3 +18,14 @@
.. branch: int_0/i-need-this-library-to-build-on-ubuntu-1-1446717626227
Document that libgdbm-dev is required for translation/packaging
+
+.. branch: propogate-nans
+
+Ensure that ndarray conversion from int16->float16->float32->float16->int16
+preserves all int16 values, even across nan conversions. Also fix argmax,
argmin
+for nan comparisons
+
+.. branch: array_interface
+
+Support common use-cases for __array_interface__, passes upstream tests
+
diff --git a/pypy/module/micronumpy/compile.py
b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -371,6 +371,8 @@
@specialize.arg(2)
def call_method(self, w_obj, s, *args):
# XXX even the hacks have hacks
+ if s == 'size': # used in _array() but never called by tests
+ return IntObject(0)
return getattr(w_obj, 'descr_' + s)(self, *args)
@specialize.arg(1)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -2,6 +2,7 @@
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
from rpython.rlib.buffer import SubBuffer
from rpython.rlib.rstring import strip_spaces
+from rpython.rlib.rawstorage import RAW_STORAGE_PTR
from rpython.rtyper.lltypesystem import lltype, rffi
from pypy.module.micronumpy import descriptor, loop, support
@@ -45,7 +46,7 @@
try:
w_interface = space.getattr(w_object,
space.wrap("__array_interface__"))
if w_interface is None:
- return None
+ return None, False
version_w = space.finditem(w_interface, space.wrap("version"))
if version_w is None:
raise oefmt(space.w_ValueError, "__array_interface__ found without"
@@ -67,19 +68,46 @@
raise oefmt(space.w_ValueError,
"__array_interface__ missing one or more required keys:
shape, typestr"
)
- raise oefmt(space.w_NotImplementedError,
- "creating array from __array_interface__ not supported
yet")
- '''
- data_w = space.listview()
+ if w_descr is not None:
+ raise oefmt(space.w_NotImplementedError,
+ "__array_interface__ descr not supported yet")
+ if w_strides is None or space.is_w(w_strides, space.w_None):
+ strides = None
+ else:
+ strides = [space.int_w(i) for i in space.listview(w_strides)]
shape = [space.int_w(i) for i in space.listview(w_shape)]
dtype = descriptor.decode_w_dtype(space, w_dtype)
- rw = space.is_true(data_w[1])
- '''
- #print 'create view from
shape',shape,'dtype',dtype,'descr',w_descr,'data',data_w[0],'rw',rw
- return None
+ if dtype is None:
+ raise oefmt(space.w_ValueError,
+ "__array_interface__ could not decode dtype %R", w_dtype
+ )
+ if w_data is not None and (space.isinstance_w(w_data, space.w_tuple)
or space.isinstance_w(w_data, space.w_list)):
+ data_w = space.listview(w_data)
+ data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+ read_only = True # XXX why not space.is_true(data_w[1])
+ offset = 0
+ return W_NDimArray.from_shape_and_storage(space, shape, data,
+ dtype, strides=strides, start=offset),
read_only
+ if w_data is None:
+ data = w_object
+ else:
+ data = w_data
+ w_offset = space.finditem(w_interface, space.wrap('offset'))
+ if w_offset is None:
+ offset = 0
+ else:
+ offset = space.int_w(w_offset)
+ #print 'create view from shape',shape,'dtype',dtype,'data',data
+ if strides is not None:
+ raise oefmt(space.w_NotImplementedError,
+ "__array_interface__ strides not fully supported yet")
+ arr = frombuffer(space, data, dtype, support.product(shape), offset)
+ new_impl = arr.implementation.reshape(arr, shape)
+ return W_NDimArray(new_impl), False
+
except OperationError as e:
if e.match(space, space.w_AttributeError):
- return None
+ return None, False
raise
@@ -103,19 +131,20 @@
if space.isinstance_w(w_object, space.w_type):
raise oefmt(space.w_ValueError, "cannot create ndarray from type
instance")
# for anything that isn't already an array, try __array__ method first
+ dtype = descriptor.decode_w_dtype(space, w_dtype)
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is not None:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
+ dtype = w_object.get_dtype()
if not isinstance(w_object, W_NDimArray):
- w_array = try_interface_method(space, w_object)
+ w_array, _copy = try_interface_method(space, w_object)
if w_array is not None:
w_object = w_array
- copy = False
- dtype = descriptor.decode_w_dtype(space, w_dtype)
-
+ copy = _copy
+ dtype = w_object.get_dtype()
if isinstance(w_object, W_NDimArray):
npy_order = order_converter(space, w_order, NPY.ANYORDER)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -534,10 +534,10 @@
while not inner_iter.done(inner_state):
arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
w_val = inner_iter.getitem(inner_state)
- new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
- if dtype.itemtype.ne(new_best, cur_best):
+ old_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+ if not old_best:
result = idx
- cur_best = new_best
+ cur_best = w_val
inner_state = inner_iter.next(inner_state)
idx += 1
result = get_dtype_cache(space).w_longdtype.box(result)
@@ -557,17 +557,17 @@
while not iter.done(state):
arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
w_val = iter.getitem(state)
- new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
- if dtype.itemtype.ne(new_best, cur_best):
+ old_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+ if not old_best:
result = idx
- cur_best = new_best
+ cur_best = w_val
state = iter.next(state)
idx += 1
return result
return argmin_argmax, argmin_argmax_flat
-argmin, argmin_flat = _new_argmin_argmax('min')
-argmax, argmax_flat = _new_argmin_argmax('max')
+argmin, argmin_flat = _new_argmin_argmax('argmin')
+argmax, argmax_flat = _new_argmin_argmax('argmax')
dot_driver = jit.JitDriver(name = 'numpy_dot',
greens = ['dtype'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py
b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1852,6 +1852,24 @@
a = array([(1, 2)], dtype=[('a', 'int64'), ('b', 'int64')])
assert a.view('S16')[0] == '\x01' + '\x00' * 7 + '\x02'
+ def test_half_conversions(self):
+ from numpy import array, arange
+ from math import isnan, isinf
+ e = array([0, -1, -float('inf'), float('nan'), 6], dtype='float16')
+ assert map(isnan, e) == [False, False, False, True, False]
+ assert map(isinf, e) == [False, False, True, False, False]
+ assert e.argmax() == 3
+ # numpy preserves value for uint16 -> cast_as_float16 ->
+ # convert_to_float64 -> convert_to_float16 -> uint16
+ # even for float16 various float16 nans
+ all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
+ all_f16.dtype = 'float16'
+ all_f32 = array(all_f16, dtype='float32')
+ b = array(all_f32, dtype='float16')
+ c = b.view(dtype='uint16')
+ d = all_f16.view(dtype='uint16')
+ assert (c == d).all()
+
def test_ndarray_view_empty(self):
from numpy import array, dtype
x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
@@ -3052,7 +3070,7 @@
assert (b == zeros(10)).all()
def test_array_interface(self):
- from numpy import array
+ from numpy import array, ones
a = array(2.5)
i = a.__array_interface__
assert isinstance(i['data'][0], int)
@@ -3075,7 +3093,7 @@
class Dummy(object):
def __init__(self, aif=None):
- if aif:
+ if aif is not None:
self.__array_interface__ = aif
a = array(Dummy())
@@ -3084,6 +3102,31 @@
raises(ValueError, array, Dummy({'version': 0}))
raises(ValueError, array, Dummy({'version': 'abc'}))
raises(ValueError, array, Dummy({'version': 3}))
+ raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8',
'shape': ('a', 3)}))
+
+ a = array([1, 2, 3])
+ b = array(Dummy(a.__array_interface__))
+ b[1] = 200
+ assert a[1] == 2 # upstream compatibility, is this a bug?
+ interface_a = a.__array_interface__
+ interface_b = b.__array_interface__
+ # only the data[0] value should differ
+ assert interface_a['data'][0] != interface_b['data'][0]
+ assert interface_b['data'][1] == interface_a['data'][1]
+ interface_b.pop('data')
+ interface_a.pop('data')
+ assert interface_a == interface_b
+
+ b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
+ 'data': 'a'*100}))
+ assert b.dtype == 'uint8'
+ assert b.shape == (50,)
+
+ a = ones((1,), dtype='float16')
+ b = Dummy(a.__array_interface__)
+ c = array(b)
+ assert c.dtype == 'float16'
+ assert (a == c).all()
def test_array_indexing_one_elem(self):
from numpy import array, arange
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -345,6 +345,14 @@
def min(self, v1, v2):
return min(v1, v2)
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ return v1 >= v2
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ return v1 <= v2
+
@raw_unary_op
def rint(self, v):
float64 = Float64(self.space)
@@ -820,6 +828,14 @@
def min(self, v1, v2):
return v1 if v1 <= v2 or rfloat.isnan(v1) else v2
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ return v1 >= v2 or rfloat.isnan(v1)
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ return v1 <= v2 or rfloat.isnan(v1)
+
@simple_binary_op
def fmax(self, v1, v2):
return v1 if v1 >= v2 or rfloat.isnan(v2) else v2
@@ -1407,6 +1423,16 @@
return v1
return v2
+ def argmin(self, v1, v2):
+ if self.le(v1, v2) or self.isnan(v1):
+ return True
+ return False
+
+ def argmax(self, v1, v2):
+ if self.ge(v1, v2) or self.isnan(v1):
+ return True
+ return False
+
@complex_binary_op
def floordiv(self, v1, v2):
(r1, i1), (r2, i2) = v1, v2
@@ -1927,6 +1953,18 @@
return v1
return v2
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ if self.space.is_true(self.space.ge(v1, v2)):
+ return True
+ return False
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ if self.space.is_true(self.space.le(v1, v2)):
+ return True
+ return False
+
@raw_unary_op
def bool(self,v):
return self._obool(v)
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -5,7 +5,8 @@
import math
from rpython.rlib import rarithmetic, rfloat, objectmodel, jit
-from rpython.rlib.rarithmetic import r_ulonglong
+from rpython.rtyper.lltypesystem.rffi import r_ulonglong, r_longlong,
LONGLONG, ULONGLONG, cast
+from rpython.rlib.longlong2float import longlong2float, float2longlong
def round_to_nearest(x):
"""Python 3 style round: round a float x to the nearest int, but
@@ -60,7 +61,20 @@
if exp == MAX_EXP - MIN_EXP + 2:
# nan or infinity
- result = rfloat.NAN if mant else rfloat.INFINITY
+ if mant == 0:
+ result = rfloat.INFINITY
+ else:
+ # preserve at most 52 bits of mant value, but pad w/zeros
+ exp = r_ulonglong(0x7ff) << 52
+ sign = r_ulonglong(sign) << 63
+ if MANT_DIG < 53:
+ mant = r_ulonglong(mant) << (53 - MANT_DIG)
+ if mant == 0:
+ result = rfloat.NAN
+ else:
+ uint = exp | mant | sign
+ result = longlong2float(cast(LONGLONG, uint))
+ return result
elif exp == 0:
# subnormal or zero
result = math.ldexp(mant, MIN_EXP - MANT_DIG)
@@ -72,7 +86,7 @@
def float_unpack80(QQ, size):
'''Unpack a (mant, exp) tuple of r_ulonglong in 80-bit extended format
- into a long double float
+ into a python float (a double)
'''
if size == 10 or size == 12 or size == 16:
MIN_EXP = -16381
@@ -100,7 +114,17 @@
if exp == MAX_EXP - MIN_EXP + 2:
# nan or infinity
- result = rfloat.NAN if mant &((one << MANT_DIG - 1) - 1) else
rfloat.INFINITY
+ if mant == 0:
+ result = rfloat.INFINITY
+ else:
+ exp = r_ulonglong(0x7ff) << 52
+ mant = r_ulonglong(mant) >> size + 1
+ if mant == 0:
+ result = rfloat.NAN
+ else:
+ uint = exp | r_ulonglong(mant) | r_ulonglong(sign)
+ result = longlong2float(cast(LONGLONG, uint))
+ return result
else:
# normal
result = math.ldexp(mant, exp + MIN_EXP - MANT_DIG - 1)
@@ -128,13 +152,19 @@
raise ValueError("invalid size value")
sign = rfloat.copysign(1.0, x) < 0.0
- if not rfloat.isfinite(x):
- if rfloat.isinf(x):
- mant = r_ulonglong(0)
- exp = MAX_EXP - MIN_EXP + 2
- else: # rfloat.isnan(x):
- mant = r_ulonglong(1) << (MANT_DIG-2) # other values possible
- exp = MAX_EXP - MIN_EXP + 2
+ if rfloat.isinf(x):
+ mant = r_ulonglong(0)
+ exp = MAX_EXP - MIN_EXP + 2
+ elif rfloat.isnan(x):
+ asint = cast(ULONGLONG, float2longlong(x))
+ sign = asint >> 63
+ # shift off lower bits, perhaps losing data
+ mant = asint & ((r_ulonglong(1) << 52) - 1)
+ if MANT_DIG < 53:
+ mant = mant >> (53 - MANT_DIG)
+ if mant == 0:
+ mant = r_ulonglong(1) << (MANT_DIG - 1) - 1
+ exp = MAX_EXP - MIN_EXP + 2
elif x == 0.0:
mant = r_ulonglong(0)
exp = 0
@@ -167,7 +197,7 @@
# check constraints
if not objectmodel.we_are_translated():
- assert 0 <= mant < 1 << MANT_DIG - 1
+ assert 0 <= mant <= (1 << MANT_DIG) - 1
assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
assert 0 <= sign <= 1
exp = r_ulonglong(exp)
@@ -187,13 +217,16 @@
raise ValueError("invalid size value")
sign = rfloat.copysign(1.0, x) < 0.0
- if not rfloat.isfinite(x):
- if rfloat.isinf(x):
- mant = r_ulonglong(0)
- exp = MAX_EXP - MIN_EXP + 2
- else: # rfloat.isnan(x):
- mant = (r_ulonglong(1) << (MANT_DIG-2)) - 1 # other values possible
- exp = MAX_EXP - MIN_EXP + 2
+ if rfloat.isinf(x):
+ mant = r_ulonglong(0)
+ exp = MAX_EXP - MIN_EXP + 2
+ elif rfloat.isnan(x): # rfloat.isnan(x):
+ asint = cast(ULONGLONG, float2longlong(x))
+ mant = asint & ((r_ulonglong(1) << 51) - 1)
+ if mant == 0:
+ mant = r_ulonglong(1) << (MANT_DIG - 1) - 1
+ sign = asint < 0
+ exp = MAX_EXP - MIN_EXP + 2
elif x == 0.0:
mant = r_ulonglong(0)
exp = 0
@@ -221,12 +254,12 @@
if exp >= MAX_EXP - MIN_EXP + 2:
raise OverflowError("float too large to pack in this format")
+ mant = mant << 1
# check constraints
if not objectmodel.we_are_translated():
- assert 0 <= mant < 1 << MANT_DIG - 1
+ assert 0 <= mant <= (1 << MANT_DIG) - 1
assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
assert 0 <= sign <= 1
- mant = mant << 1
exp = r_ulonglong(exp)
sign = r_ulonglong(sign)
return (mant, (sign << BITS - MANT_DIG - 1) | exp)
diff --git a/rpython/rlib/rstruct/test/test_ieee.py
b/rpython/rlib/rstruct/test/test_ieee.py
--- a/rpython/rlib/rstruct/test/test_ieee.py
+++ b/rpython/rlib/rstruct/test/test_ieee.py
@@ -168,15 +168,31 @@
def test_random(self):
# construct a Python float from random integer, using struct
+ mantissa_mask = (1 << 53) - 1
for _ in xrange(10000):
Q = random.randrange(2**64)
x = struct.unpack('<d', struct.pack('<Q', Q))[0]
# nans are tricky: we can't hope to reproduce the bit
- # pattern exactly, so check_float will fail for a random nan.
- if isnan(x):
+ # pattern exactly, so check_float will fail for a nan
+ # whose mantissa does not fit into float16's mantissa.
+ if isnan(x) and (Q & mantissa_mask) >= 1 << 11:
continue
self.check_float(x)
+ def test_various_nans(self):
+ # check patterns that should preserve the mantissa across nan
conversions
+ maxmant64 = (1 << 52) - 1 # maximum double mantissa
+ maxmant16 = (1 << 10) - 1 # maximum float16 mantissa
+ assert maxmant64 >> 42 == maxmant16
+ exp = 0xfff << 52
+ for i in range(20):
+ val_to_preserve = exp | ((maxmant16 - i) << 42)
+ a = ieee.float_unpack(val_to_preserve, 8)
+ assert isnan(a), 'i %d, maxmant %s' % (i, hex(val_to_preserve))
+ b = ieee.float_pack(a, 8)
+ assert b == val_to_preserve, 'i %d, val %s b %s' % (i,
hex(val_to_preserve), hex(b))
+ b = ieee.float_pack(a, 2)
+ assert b == 0xffff - i, 'i %d, b%s' % (i, hex(b))
class TestCompiled:
def test_pack_float(self):
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
