Author: Armin Rigo <[email protected]>
Branch:
Changeset: r92033:1903d256e24b
Date: 2017-08-02 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/1903d256e24b/
Log: import cffi/120347b84c08
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -394,12 +394,17 @@
replace_with = ' ' + replace_with
return self._backend.getcname(cdecl, replace_with)
- def gc(self, cdata, destructor):
+ def gc(self, cdata, destructor, size=0):
"""Return a new cdata object that points to the same
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
+
+ The optional 'size' gives an estimate of the size, used to
+ trigger the garbage collection more eagerly. So far only used
+ on PyPy. It tells the GC that the returned object keeps alive
+ roughly 'size' bytes of external memory.
"""
- return self._backend.gcp(cdata, destructor)
+ return self._backend.gcp(cdata, destructor, size)
def _get_cached_btype(self, type):
assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
_weakref_cache_ref = None
- def gcp(self, cdata, destructor):
+ def gcp(self, cdata, destructor, size=0):
if self._weakref_cache_ref is None:
import weakref
class MyRef(weakref.ref):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2455,3 +2455,61 @@
assert (pt.x, pt.y) == (-9*500*999, 9*500*999)
pt = lib.call2(lib.cb2)
assert (pt.x, pt.y) == (99*500*999, -99*500*999)
+
+def test_ffi_gc_size_arg():
+ # with PyPy's GC, these calls to ffi.gc() would rapidly consume
+ # 40 GB of RAM without the third argument
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ for i in range(2000):
+ p = lib.malloc(20*1024*1024) # 20 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 20*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 20*1024*1024)
+ del p
+
+def test_ffi_gc_size_arg_2():
+ # a variant of the above: this "attack" works on cpython's cyclic gc too
+ # and I found no obvious way to prevent that. So for now, this test
+ # is skipped on CPython, where it eats all the memory.
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ class X(object):
+ pass
+ for i in range(2000):
+ p = lib.malloc(50*1024*1024) # 50 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 50*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 50*1024*1024)
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
+
+def test_ffi_new_with_cycles():
+ # still another variant, with ffi.new()
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("")
+ lib = ffi.verify("")
+ class X(object):
+ pass
+ for i in range(2000):
+ p = ffi.new("char[]", 50*1024*1024) # 50 MB
+ for j in xrange(0, 50*1024*1024, 4096):
+ p[j] = '!'
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2291,3 +2291,61 @@
expected = "unsigned int"
assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
+
+def test_gc_pypy_size_arg():
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ for i in range(2000):
+ p = lib.malloc(20*1024*1024) # 20 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 20*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 20*1024*1024)
+ del p
+ # with PyPy's GC, the above would rapidly consume 40 GB of RAM
+ # without the third argument to ffi.gc()
+
+def test_ffi_gc_size_arg_2():
+ # a variant of the above: this "attack" works on cpython's cyclic gc too
+ # and I found no obvious way to prevent that. So for now, this test
+ # is skipped on CPython, where it eats all the memory.
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("void *malloc(size_t); void free(void *);")
+ lib = ffi.verify(r"""
+ #include <stdlib.h>
+ """)
+ class X(object):
+ pass
+ for i in range(2000):
+ p = lib.malloc(50*1024*1024) # 50 MB
+ p1 = ffi.cast("char *", p)
+ for j in xrange(0, 50*1024*1024, 4096):
+ p1[j] = '!'
+ p = ffi.gc(p, lib.free, 50*1024*1024)
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
+
+def test_ffi_new_with_cycles():
+ # still another variant, with ffi.new()
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("find a way to tweak the cyclic GC of CPython")
+ ffi = FFI()
+ ffi.cdef("")
+ lib = ffi.verify("")
+ class X(object):
+ pass
+ for i in range(2000):
+ p = ffi.new("char[]", 50*1024*1024) # 50 MB
+ for j in xrange(0, 50*1024*1024, 4096):
+ p[j] = '!'
+ x = X()
+ x.p = p
+ x.cyclic = x
+ del p, x
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
