Author: Carl Friedrich Bolz <[email protected]>
Branch: getarrayitem-into-bridges
Changeset: r92042:d16b10f8ca91
Date: 2017-08-02 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d16b10f8ca91/
Log: merge default
diff --git a/lib-python/2.7/distutils/unixccompiler.py
b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
return "-L" + dir
def _is_gcc(self, compiler_name):
- return "gcc" in compiler_name or "g++" in compiler_name
+ # XXX PyPy workaround, look at the big comment below for more
+ # context. On CPython, the hack below works fine because
+ # `compiler_name` contains the name of the actual compiler which was
+ # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+ # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+ # result is that we pass the wrong option to the compiler.
+ #
+ # The workaround is to *always* pretend to be GCC if we are on Linux:
+ # this should cover the vast majority of real systems, including the
+ # ones which use clang (which understands the '-Wl,-rpath' syntax as
+ # well)
+ return (sys.platform == "linux2" or
+ "gcc" in compiler_name or "g++" in compiler_name)
def runtime_library_dir_option(self, dir):
# XXX Hackish, at the very least. See Python bug #445902:
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -394,12 +394,17 @@
replace_with = ' ' + replace_with
return self._backend.getcname(cdecl, replace_with)
- def gc(self, cdata, destructor):
+ def gc(self, cdata, destructor, size=0):
"""Return a new cdata object that points to the same
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
+
+ The optional 'size' gives an estimate of the size, used to
+ trigger the garbage collection more eagerly. So far only used
+ on PyPy. It tells the GC that the returned object keeps alive
+ roughly 'size' bytes of external memory.
"""
- return self._backend.gcp(cdata, destructor)
+ return self._backend.gcp(cdata, destructor, size)
def _get_cached_btype(self, type):
assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
_weakref_cache_ref = None
- def gcp(self, cdata, destructor):
+ def gcp(self, cdata, destructor, size=0):
if self._weakref_cache_ref is None:
import weakref
class MyRef(weakref.ref):
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -55,3 +55,8 @@
Fix the bounds in the GC when allocating a lot of objects with finalizers,
fixes issue #2590
+
+.. branch: arrays-force-less
+
+Small improvement to optimize list accesses with constant indexes better by
+throwing away information about them less eagerly.
diff --git a/pypy/module/_cffi_backend/cdataobj.py
b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -433,17 +433,22 @@
def _sizeof(self):
return self.ctype.size
- def with_gc(self, w_destructor):
+ def with_gc(self, w_destructor, size=0):
space = self.space
if space.is_none(w_destructor):
if isinstance(self, W_CDataGCP):
self.detach_destructor()
- return space.w_None
- raise oefmt(space.w_TypeError,
- "Can remove destructor only on a object "
- "previously returned by ffi.gc()")
- with self as ptr:
- return W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
+ w_res = space.w_None
+ else:
+ raise oefmt(space.w_TypeError,
+ "Can remove destructor only on a object "
+ "previously returned by ffi.gc()")
+ else:
+ with self as ptr:
+ w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
+ if size != 0:
+ rgc.add_memory_pressure(size)
+ return w_res
def unpack(self, length):
from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
diff --git a/pypy/module/_cffi_backend/ffi_obj.py
b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -351,14 +351,14 @@
return handle.from_handle(self.space, w_arg)
- @unwrap_spec(w_cdata=W_CData)
- def descr_gc(self, w_cdata, w_destructor):
+ @unwrap_spec(w_cdata=W_CData, size=int)
+ def descr_gc(self, w_cdata, w_destructor, size=0):
"""\
Return a new cdata object that points to the same data.
Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called."""
#
- return w_cdata.with_gc(w_destructor)
+ return w_cdata.with_gc(w_destructor, size)
@unwrap_spec(replace_with='text')
diff --git a/pypy/module/_cffi_backend/func.py
b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -257,6 +257,6 @@
# ____________________________________________________________
-@unwrap_spec(w_cdata=cdataobj.W_CData)
-def gcp(space, w_cdata, w_destructor):
- return w_cdata.with_gc(w_destructor)
+@unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
+def gcp(space, w_cdata, w_destructor, size=0):
+ return w_cdata.with_gc(w_destructor, size)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -377,7 +377,7 @@
raises(TypeError, ffi.gc, p, None)
seen = []
q1 = ffi.gc(p, lambda p: seen.append(1))
- q2 = ffi.gc(q1, lambda p: seen.append(2))
+ q2 = ffi.gc(q1, lambda p: seen.append(2), size=123)
import gc; gc.collect()
assert seen == []
assert ffi.gc(q1, None) is None
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2455,3 +2455,61 @@
assert (pt.x, pt.y) == (-9*500*999, 9*500*999)
pt = lib.call2(lib.cb2)
assert (pt.x, pt.y) == (99*500*999, -99*500*999)
+
+def test_ffi_gc_size_arg():
+ # with PyPy's GC, these calls to ffi.gc() would rapidly consume
+ # 40 GB of RAM without the third argument
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ for i in range(2000):
+ p = lib.malloc(20*1024*1024) # 20 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 20*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 20*1024*1024)
+ del p
+
+def test_ffi_gc_size_arg_2():
+ # a variant of the above: this "attack" works on cpython's cyclic gc too
+ # and I found no obvious way to prevent that. So for now, this test
+ # is skipped on CPython, where it eats all the memory.
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ class X(object):
+ pass
+ for i in range(2000):
+ p = lib.malloc(50*1024*1024) # 50 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 50*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 50*1024*1024)
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
+
+def test_ffi_new_with_cycles():
+ # still another variant, with ffi.new()
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("")
+ lib = ffi.verify("")
+ class X(object):
+ pass
+ for i in range(2000):
+ p = ffi.new("char[]", 50*1024*1024) # 50 MB
+ for j in xrange(0, 50*1024*1024, 4096):
+ p[j] = '!'
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2291,3 +2291,61 @@
expected = "unsigned int"
assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
+
+def test_gc_pypy_size_arg():
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ for i in range(2000):
+ p = lib.malloc(20*1024*1024) # 20 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 20*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 20*1024*1024)
+ del p
+ # with PyPy's GC, the above would rapidly consume 40 GB of RAM
+ # without the third argument to ffi.gc()
+
+def test_ffi_gc_size_arg_2():
+ # a variant of the above: this "attack" works on cpython's cyclic gc too
+ # and I found no obvious way to prevent that. So for now, this test
+ # is skipped on CPython, where it eats all the memory.
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ class X(object):
+ pass
+ for i in range(2000):
+ p = lib.malloc(50*1024*1024) # 50 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 50*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 50*1024*1024)
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
+
+def test_ffi_new_with_cycles():
+ # still another variant, with ffi.new()
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("")
+ lib = ffi.verify("")
+ class X(object):
+ pass
+ for i in range(2000):
+ p = ffi.new("char[]", 50*1024*1024) # 50 MB
+ for j in xrange(0, 50*1024*1024, 4096):
+ p[j] = '!'
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
