Author: Carl Friedrich Bolz <[email protected]>
Branch: value-profiling
Changeset: r78993:f22171f122f5
Date: 2015-08-15 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/f22171f122f5/
Log: merge default
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -66,3 +66,4 @@
.. branch: vmprof-review
+Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,4 +1,5 @@
from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rvmprof import VMProfPlatformUnsupported
class Module(MixedModule):
"""
@@ -19,4 +20,7 @@
# already found by the annotator to be the original empty
# method, and the annotator doesn't notice that interp_vmprof.py
# (loaded later) replaces this method.
-import pypy.module._vmprof.interp_vmprof
+try:
+ import pypy.module._vmprof.interp_vmprof
+except VMProfPlatformUnsupported, e:
+ pass
diff --git a/pypy/module/micronumpy/concrete.py
b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -52,11 +52,16 @@
@jit.unroll_safe
def setslice(self, space, arr):
- if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
- raise oefmt(space.w_ValueError,
- "could not broadcast input array from shape "
- "(%s) into shape ()",
- ','.join([str(x) for x in arr.get_shape()]))
+ if len(arr.get_shape()) > len(self.get_shape()):
+ # record arrays get one extra dimension
+ if not self.dtype.is_record() or \
+ len(arr.get_shape()) > len(self.get_shape()) + 1:
+ raise oefmt(space.w_ValueError,
+ "could not broadcast input array from shape "
+ "(%s) into shape (%s)",
+ ','.join([str(x) for x in arr.get_shape()]),
+ ','.join([str(x) for x in self.get_shape()]),
+ )
shape = shape_agreement(space, self.get_shape(), arr)
impl = arr.implementation
if impl.storage == self.storage:
diff --git a/pypy/module/micronumpy/ndarray.py
b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -247,6 +247,20 @@
if space.is_w(w_idx, space.w_Ellipsis):
self.implementation.setslice(space, convert_to_array(space,
w_value))
return
+ # TODO: multiarray/mapping.c calls a subclass's __getitem__ here, which
+ # is a big performance hit but necessary for the matrix class. The
original
+ # C code is like:
+ #/*
+ #* WARNING: There is a huge special case here. If this is not a
+ #* base class array, we have to get the view through its
+ #* very own index machinery.
+ #* Many subclasses should probably call __setitem__
+ #* with a base class ndarray view to avoid this.
+ #*/
+ #else if (!(index_type & (HAS_FANCY | HAS_SCALAR_ARRAY))
+ # && !PyArray_CheckExact(self)) {
+ #view = (PyArrayObject *)PyObject_GetItem((PyObject *)self, ind);
+
elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
and w_idx.ndims() > 0:
self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
diff --git a/pypy/module/micronumpy/test/test_ndarray.py
b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2668,6 +2668,15 @@
a[0, :: -1] = numpy.array([11, 12])
assert (a == [[12, 11], [5, 10]]).all()
+ a = numpy.zeros((3, 2), int)
+ b = numpy.ones((3, 1), int)
+ exc = raises(ValueError, 'a[:, 1] = b')
+ assert str(exc.value) == "could not broadcast " +\
+ "input array from shape (3,1) into shape (3)"
+ a[:, 1] = b[:,0] > 0.5
+ assert (a == [[0, 1], [0, 1], [0, 1]]).all()
+
+
def test_ufunc(self):
from numpy import array
a = array([[1, 2], [3, 4], [5, 6]])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py
b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1166,6 +1166,7 @@
assert (logical_xor([True, False, True, False], [1, 2, 0, 0])
== [False, True, True, False]).all()
assert (logical_not([True, False]) == [False, True]).all()
+ assert logical_and.reduce([1.,1.]) == True
def test_logn(self):
import math
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -511,15 +511,15 @@
W_Ufunc.__init__(self, name, promote_to_largest, promote_to_float,
promote_bools,
identity, int_only, allow_bool, allow_complex,
complex_to_float)
self.func = func
- self.bool_result = bool_result
if name == 'logical_and':
self.done_func = done_if_false
elif name == 'logical_or':
self.done_func = done_if_true
else:
self.done_func = None
+ self.bool_result = bool_result or (self.done_func is not None)
self.simple_binary = (
- allow_complex and allow_bool and not bool_result and not int_only
+ allow_complex and allow_bool and not self.bool_result and not
int_only
and not complex_to_float and not promote_to_float
and not promote_bools)
@@ -630,7 +630,7 @@
r_dtype.is_complex())):
raise oefmt(space.w_TypeError,
"ufunc '%s' not supported for the input types", self.name)
- if self.bool_result:
+ if self.bool_result and not self.done_func:
# XXX: should actually pass the arrays
dtype = find_result_type(space, [], [l_dtype, r_dtype])
bool_dtype = get_dtype_cache(space).w_booldtype
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -28,9 +28,9 @@
f16 = raw_load(i9, i5, descr=<ArrayF \d+>)
guard_true(i15, descr=...)
guard_not_invalidated(descr=...)
- i17 = cast_float_to_int(f16)
- i19 = int_is_true(i17)
- guard_true(i19, descr=...)
+ i18 = float_ne(f16, 0.000000)
+ guard_true(i18, descr=...)
+ guard_nonnull_class(p2, ConstClass(W_BoolBox), descr=...)
i20 = getfield_gc_pure(p2, descr=<FieldU
pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
i21 = int_is_true(i20)
guard_false(i21, descr=...)
@@ -70,10 +70,10 @@
assert loop.match("""
f31 = raw_load(i9, i29, descr=<ArrayF 8>)
guard_not_invalidated(descr=...)
+ i32 = float_ne(f31, 0.000000)
+ guard_true(i32, descr=...)
i34 = getarrayitem_raw(#, #, descr=<ArrayU 1>) # XXX what are
these?
guard_value(i34, #, descr=...) # XXX don't appear
in
- i32 = float_ne(f31, 0.000000)
- guard_true(i32, descr=...)
i35 = getarrayitem_raw(#, #, descr=<ArrayU 1>) # XXX equiv
test_zjit
i36 = int_add(i24, 1)
i37 = int_add(i29, i28)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -569,17 +569,19 @@
rangen = unroll.unrolling_iterable(range(n))
nmin1 = n - 1
rangenmin1 = unroll.unrolling_iterable(range(nmin1))
+ valnmin1 = "_value%s" % nmin1
class subcls(BaseMapdictObject, supercls):
def _init_empty(self, map):
- for i in rangen:
- setattr(self, "_value%s" % i, erase_item(None))
+ for i in rangenmin1:
+ setattr(self, "_value%s" % i, None)
+ setattr(self, valnmin1, erase_item(None))
self.map = map
def _has_storage_list(self):
return self.map.length() > n
def _mapdict_get_storage_list(self):
- erased = getattr(self, "_value%s" % nmin1)
+ erased = getattr(self, valnmin1)
return unerase_list(erased)
def _mapdict_read_storage(self, storageindex):
@@ -587,23 +589,21 @@
if storageindex < nmin1:
for i in rangenmin1:
if storageindex == i:
- erased = getattr(self, "_value%s" % i)
- return unerase_item(erased)
+ return getattr(self, "_value%s" % i)
if self._has_storage_list():
return self._mapdict_get_storage_list()[storageindex - nmin1]
erased = getattr(self, "_value%s" % nmin1)
return unerase_item(erased)
def _mapdict_write_storage(self, storageindex, value):
- erased = erase_item(value)
for i in rangenmin1:
if storageindex == i:
- setattr(self, "_value%s" % i, erased)
+ setattr(self, "_value%s" % i, value)
return
if self._has_storage_list():
self._mapdict_get_storage_list()[storageindex - nmin1] = value
return
- setattr(self, "_value%s" % nmin1, erased)
+ setattr(self, "_value%s" % nmin1, erase_item(value))
def _mapdict_storage_length(self):
if self._has_storage_list():
@@ -615,9 +615,9 @@
len_storage = len(storage)
for i in rangenmin1:
if i < len_storage:
- erased = erase_item(storage[i])
+ erased = storage[i]
else:
- erased = erase_item(None)
+ erased = None
setattr(self, "_value%s" % i, erased)
has_storage_list = self._has_storage_list()
if len_storage < n:
diff --git a/pypy/objspace/std/stringmethods.py
b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -195,7 +195,8 @@
splitted = split(value, self._chr('\t'))
try:
- ovfcheck(len(splitted) * tabsize)
+ if tabsize > 0:
+ ovfcheck(len(splitted) * tabsize)
except OverflowError:
raise oefmt(space.w_OverflowError, "new string is too long")
expanded = oldtoken = splitted.pop(0)
@@ -210,6 +211,8 @@
def _tabindent(self, token, tabsize):
"""calculates distance behind the token to the next tabstop"""
+ if tabsize <= 0:
+ return 0
distance = tabsize
if token:
distance = 0
diff --git a/pypy/objspace/std/test/test_bytesobject.py
b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -388,6 +388,10 @@
skip("Wrong platform")
raises((MemoryError, OverflowError), 't\tt\t'.expandtabs, sys.maxint)
+ def test_expandtabs_0(self):
+ assert 'x\ty'.expandtabs(0) == 'xy'
+ assert 'x\ty'.expandtabs(-42) == 'xy'
+
def test_splitlines(self):
s = ""
assert s.splitlines() == []
diff --git a/pypy/objspace/std/test/test_mapdict.py
b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -500,12 +500,12 @@
obj = objectcls()
obj.user_setup(space, cls)
obj.setdictvalue(space, "a", w1)
- assert unerase_item(obj._value0) is w1
+ assert obj._value0 is w1
assert obj.getdictvalue(space, "a") is w1
assert obj.getdictvalue(space, "b") is None
assert obj.getdictvalue(space, "c") is None
obj.setdictvalue(space, "a", w2)
- assert unerase_item(obj._value0) is w2
+ assert obj._value0 is w2
assert obj.getdictvalue(space, "a") == w2
assert obj.getdictvalue(space, "b") is None
assert obj.getdictvalue(space, "c") is None
@@ -523,7 +523,7 @@
res = obj.deldictvalue(space, "a")
assert res
- assert unerase_item(obj._value0) is w4
+ assert obj._value0 is w4
assert obj.getdictvalue(space, "a") is None
assert obj.getdictvalue(space, "b") is w4
assert obj.getdictvalue(space, "c") is None
diff --git a/pypy/objspace/std/test/test_unicodeobject.py
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -494,6 +494,10 @@
skip("Wrong platform")
raises((OverflowError, MemoryError), u't\tt\t'.expandtabs, sys.maxint)
+ def test_expandtabs_0(self):
+ assert u'x\ty'.expandtabs(0) == u'xy'
+ assert u'x\ty'.expandtabs(-42) == u'xy'
+
def test_translate(self):
assert u'bbbc' == u'abababc'.translate({ord('a'):None})
assert u'iiic' == u'abababc'.translate({ord('a'):None,
ord('b'):ord('i')})
diff --git a/rpython/jit/codewriter/support.py
b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -79,9 +79,6 @@
assert methname == 'jit_merge_point', (
"reds='auto' is supported only for jit drivers which "
"calls only jit_merge_point. Found a call to %s" % methname)
- if jitdriver.numreds is not None:
- raise AssertionError("there are multiple jit_merge_points "
- "with the same jitdriver")
#
# compute the set of live variables across the jit_marker
alive_v = set()
@@ -99,7 +96,11 @@
v.concretetype is not lltype.Void]
reds_v = sort_vars(reds_v)
op.args.extend(reds_v)
- jitdriver.numreds = len(reds_v)
+ if jitdriver.numreds is None:
+ jitdriver.numreds = len(reds_v)
+ elif jitdriver.numreds != len(reds_v):
+ raise AssertionError("there are multiple jit_merge_points "
+ "with the same jitdriver")
def split_before_jit_merge_point(graph, portalblock, portalopindex):
"""Split the block just before the 'jit_merge_point',
diff --git a/rpython/jit/metainterp/heapcache.py
b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -9,6 +9,7 @@
def reset_keep_likely_virtual(self):
self.known_class = False
+ self.known_nullity = False
# did we see the allocation during tracing?
self.seen_allocation = False
self.is_unescaped = False
@@ -290,6 +291,15 @@
def class_now_known(self, box):
self.getvalue(box).known_class = True
+ def is_nullity_known(self, box):
+ value = self.getvalue(box, create=False)
+ if value:
+ return value.known_nullity
+ return False
+
+ def nullity_now_known(self, box):
+ self.getvalue(box).known_nullity = True
+
def is_nonstandard_virtualizable(self, box):
value = self.getvalue(box, create=False)
if value:
diff --git a/rpython/jit/metainterp/pyjitpl.py
b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -369,7 +369,10 @@
).compile()
def _establish_nullity(self, box, orgpc):
+ heapcache = self.metainterp.heapcache
value = box.nonnull()
+ if heapcache.is_nullity_known(box):
+ return value
if value:
if not self.metainterp.heapcache.is_class_known(box):
self.metainterp.generate_guard(rop.GUARD_NONNULL, box,
@@ -380,6 +383,7 @@
resumepc=orgpc)
promoted_box = box.constbox()
self.metainterp.replace_box(box, promoted_box)
+ heapcache.nullity_now_known(box)
return value
@arguments("box", "label", "orgpc")
diff --git a/rpython/jit/metainterp/test/test_heapcache.py
b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -69,6 +69,19 @@
assert not h.is_class_known(1)
assert not h.is_class_known(2)
+ def test_known_nullity(self):
+ h = HeapCache()
+ assert not h.is_nullity_known(1)
+ assert not h.is_nullity_known(2)
+ h.nullity_now_known(1)
+ assert h.is_nullity_known(1)
+ assert not h.is_nullity_known(2)
+
+ h.reset()
+ assert not h.is_nullity_known(1)
+ assert not h.is_nullity_known(2)
+
+
def test_nonstandard_virtualizable(self):
h = HeapCache()
assert not h.is_nonstandard_virtualizable(1)
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py
b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -107,6 +107,28 @@
assert res == -7 * 2
self.check_operations_history(getfield_gc=1)
+ def test_heap_caching_nonnull(self):
+ class A:
+ def __init__(self, x=None):
+ self.next = x
+ a0 = A()
+ a1 = A()
+ a2 = A(a1)
+ def fn(n):
+ if n > 0:
+ a = a1
+ else:
+ a = a2
+ if a.next:
+ a = A(a.next)
+ result = a.next is not None
+ a0.next = a
+ return result
+ return False
+ res = self.interp_operations(fn, [-7])
+ assert res == True
+ self.check_operations_history(guard_nonnull=1)
+
def test_heap_caching_while_tracing_invalidation(self):
class A:
pass
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -725,10 +725,15 @@
result_w = []
#
if not we_are_translated(): # fast path before translation
- for gcref in roots: # 'roots' is all objects in this case
- w_obj = callback(gcref)
- if w_obj is not None:
- result_w.append(w_obj)
+ seen = set()
+ while roots:
+ gcref = roots.pop()
+ if gcref not in seen:
+ seen.add(gcref)
+ w_obj = callback(gcref)
+ if w_obj is not None:
+ result_w.append(w_obj)
+ roots.extend(get_rpy_referents(gcref))
return result_w
#
pending = roots[:]
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -2,6 +2,7 @@
from rpython.rlib.rvmprof.rvmprof import _get_vmprof, VMProfError
from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME
from rpython.rlib.rvmprof.rvmprof import _was_registered
+from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported
#
# See README.txt.
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -1,6 +1,7 @@
import py
import sys
from rpython.tool.udir import udir
+from rpython.tool.version import rpythonroot
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rtyper.tool import rffi_platform as platform
@@ -16,7 +17,7 @@
" x86-64 CPUs for now")
- ROOT = py.path.local(__file__).join('..')
+ ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
SRC = ROOT.join('src')
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -196,6 +196,11 @@
(including 'self' if applicable).
"""
def decorate(func):
+ try:
+ _get_vmprof()
+ except cintf.VMProfPlatformUnsupported:
+ return func
+
if hasattr(func, 'im_self'):
assert func.im_self is None
func = func.im_func
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py
b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -8,7 +8,10 @@
class MyCode:
pass
- rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ try:
+ rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ except rvmprof.VMProfPlatformUnsupported:
+ pass
@rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
def main(code, num):
@@ -29,7 +32,10 @@
class MyCode:
pass
- rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ try:
+ rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ except rvmprof.VMProfPlatformUnsupported:
+ pass
class A:
pass
@@ -54,7 +60,10 @@
class MyCode:
pass
- rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ try:
+ rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+ except rvmprof.VMProfPlatformUnsupported, e:
+ py.test.skip(str(e))
@rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
def main(code, num):
@@ -79,7 +88,10 @@
pass
def get_name(code):
return 'py:code:52:x'
- rvmprof.register_code_object_class(MyCode, get_name)
+ try:
+ rvmprof.register_code_object_class(MyCode, get_name)
+ except rvmprof.VMProfPlatformUnsupported, e:
+ py.test.skip(str(e))
@rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
def main(code, num):
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py
b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,4 @@
-import time, os
+import time, os, sys, py
from rpython.tool.udir import udir
from rpython.rlib import rvmprof
from rpython.translator.c.test.test_genc import compile
@@ -14,7 +14,10 @@
def setup_module(mod):
- rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+ try:
+ rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+ except rvmprof.VMProfPlatformUnsupported, e:
+ py.test.skip(str(e))
@rvmprof.vmprof_execute_code("interp", lambda code: code)
@@ -52,7 +55,9 @@
return main
def test_interpreted():
- main()
+ # takes forever if the Python process is already big...
+ import subprocess
+ subprocess.check_call([sys.executable, __file__])
def test_compiled():
fn = compile(main, [], gcpolicy="minimark")
@@ -60,3 +65,8 @@
os.unlink(PROF_FILE)
fn()
assert os.path.exists(PROF_FILE)
+
+if __name__ == '__main__':
+ setup_module(None)
+ res = main()
+ assert res == 0
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
