Author: fijal
Branch:
Changeset: r93768:f23eec5d0d6d
Date: 2018-02-06 10:49 +0100
http://bitbucket.org/pypy/pypy/changeset/f23eec5d0d6d/
Log: Merge memory-accounting which adds extra functions to gc that let
you describe the whole memory
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -35,7 +35,7 @@
except ImportError:
assert '__pypy__' not in sys.builtin_module_names
newlist_hint = lambda sizehint: []
- add_memory_pressure = lambda size: None
+ add_memory_pressure = lambda size, obj: None
if sys.version_info[0] >= 3:
StandardError = Exception
@@ -153,9 +153,10 @@
factory = Connection if not factory else factory
# an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
# backed by :memory: or a file)
- add_memory_pressure(100 * 1024)
- return factory(database, timeout, detect_types, isolation_level,
+ res = factory(database, timeout, detect_types, isolation_level,
check_same_thread, factory, cached_statements)
+ add_memory_pressure(100 * 1024, res)
+ return res
def _unicode_text_factory(x):
diff --git a/pypy/module/__pypy__/interp_magic.py
b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -142,11 +142,14 @@
space.newbool(debug))
@unwrap_spec(estimate=int)
-def add_memory_pressure(estimate):
+def add_memory_pressure(space, estimate, w_obj=None):
""" Add memory pressure of estimate bytes. Useful when calling a C function
that internally allocates a big chunk of memory. This instructs the GC to
garbage collect sooner than it would otherwise."""
+ #if space.is_none(w_obj):
rgc.add_memory_pressure(estimate)
+ #else:
+ # rgc.add_memory_pressure(estimate, w_obj)
@unwrap_spec(w_frame=PyFrame)
def locals_to_fast(space, w_frame):
diff --git a/pypy/module/_cffi_backend/allocator.py
b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -21,13 +21,13 @@
if self.w_alloc is None:
if self.should_clear_after_alloc:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=True,
- add_memory_pressure=True)
+ flavor='raw', zero=True)
else:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=False,
- add_memory_pressure=True)
- return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ flavor='raw', zero=False)
+ w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ rgc.add_memory_pressure(datasize, w_res)
+ return w_res
else:
w_raw_cdata = space.call_function(self.w_alloc,
space.newint(datasize))
@@ -53,7 +53,7 @@
if self.w_free is not None:
res.w_free = self.w_free
res.register_finalizer(space)
- rgc.add_memory_pressure(datasize)
+ rgc.add_memory_pressure(datasize, res)
return res
@unwrap_spec(w_init=WrappedDefault(None))
diff --git a/pypy/module/_cffi_backend/cdataobj.py
b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -447,7 +447,10 @@
with self as ptr:
w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
if size != 0:
- rgc.add_memory_pressure(size)
+ if isinstance(w_res, W_CDataGCP):
+ rgc.add_memory_pressure(size, w_res)
+ else:
+ rgc.add_memory_pressure(size, self)
return w_res
def unpack(self, length):
diff --git a/pypy/module/_hashlib/interp_hashlib.py
b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -61,7 +61,8 @@
ctx = ropenssl.EVP_MD_CTX_new()
if ctx is None:
raise MemoryError
- rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
+ rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size,
+ self)
try:
if copy_from:
if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1316,8 +1316,9 @@
if not ctx:
raise ssl_error(space, "failed to allocate SSL context")
- rgc.add_memory_pressure(10 * 1024 * 1024)
self = space.allocate_instance(_SSLContext, w_subtype)
+ assert isinstance(self, _SSLContext)
+ rgc.add_memory_pressure(10 * 1024 * 1024, self)
self.ctx = ctx
self.check_hostname = False
self.register_finalizer(space)
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -19,6 +19,7 @@
space.config.translation.gctransformer == "framework"):
self.appleveldefs.update({
'dump_rpy_heap': 'app_referents.dump_rpy_heap',
+ 'get_stats': 'app_referents.get_stats',
})
self.interpleveldefs.update({
'get_rpy_roots': 'referents.get_rpy_roots',
@@ -28,6 +29,7 @@
'get_objects': 'referents.get_objects',
'get_referents': 'referents.get_referents',
'get_referrers': 'referents.get_referrers',
+ '_get_stats': 'referents.get_stats',
'_dump_rpy_heap': 'referents._dump_rpy_heap',
'get_typeids_z': 'referents.get_typeids_z',
'get_typeids_list': 'referents.get_typeids_list',
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -48,3 +48,49 @@
file.flush()
fd = file.fileno()
gc._dump_rpy_heap(fd)
+
+class GcStats(object):
+ def __init__(self, s):
+ self._s = s
+ for item in ('total_gc_memory', 'jit_backend_used',
+ 'total_memory_pressure',
+ 'total_allocated_memory', 'jit_backend_allocated',
+ 'peak_memory', 'peak_allocated_memory'):
+ setattr(self, item, self._format(getattr(self._s, item)))
+ self.memory_used_sum = self._format(self._s.total_gc_memory +
self._s.total_memory_pressure +
+ self._s.jit_backend_used)
+ self.memory_allocated_sum =
self._format(self._s.total_allocated_memory + self._s.total_memory_pressure +
+ self._s.jit_backend_allocated)
+
+ def _format(self, v):
+ if v < 1000000:
+ # bit unlikely ;-)
+ return "%.1fkB" % (v / 1024.)
+ return "%.1fMB" % (v / 1024. / 1024.)
+
+ def repr(self):
+ return """Total memory consumed:
+GC used: %s (peak: %s)
+raw assembler used: %s
+memory pressure: %s
+-----------------------------
+Total: %s
+
+Total memory allocated:
+GC allocated: %s (peak: %s)
+raw assembler allocated: %s
+memory pressure: %s
+-----------------------------
+Total: %s
+""" % (self.total_gc_memory, self.peak_memory,
+ self.jit_backend_used,
+ self.total_memory_pressure,
+ self.memory_used_sum,
+
+ self.total_allocated_memory, self.peak_allocated_memory,
+ self.jit_backend_allocated,
+ self.total_memory_pressure,
+ self.memory_allocated_sum)
+
+def get_stats():
+ return GcStats(gc._get_stats())
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -1,7 +1,7 @@
-from rpython.rlib import rgc
+from rpython.rlib import rgc, jit_hooks
from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.gateway import unwrap_spec, interp2app
from pypy.interpreter.error import oefmt, wrap_oserror
from rpython.rlib.objectmodel import we_are_translated
@@ -170,3 +170,33 @@
l = rgc.get_typeids_list()
list_w = [space.newint(l[i]) for i in range(len(l))]
return space.newlist(list_w)
+
+class W_GcStats(W_Root):
+ def __init__(self):
+ self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+ self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY)
+ self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY)
+ self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY)
+ self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY)
+ self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None)
+ self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None)
+
+W_GcStats.typedef = TypeDef("GcStats",
+ total_memory_pressure=interp_attrproperty("total_memory_pressure",
+ cls=W_GcStats, wrapfn="newint"),
+ total_gc_memory=interp_attrproperty("total_gc_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ peak_allocated_memory=interp_attrproperty("peak_allocated_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ peak_memory=interp_attrproperty("peak_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ total_allocated_memory=interp_attrproperty("total_allocated_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ jit_backend_allocated=interp_attrproperty("jit_backend_allocated",
+ cls=W_GcStats, wrapfn="newint"),
+ jit_backend_used=interp_attrproperty("jit_backend_used",
+ cls=W_GcStats, wrapfn="newint"),
+)
+
+def get_stats(space):
+ return W_GcStats()
diff --git a/pypy/module/pyexpat/interp_pyexpat.py
b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -840,11 +840,11 @@
# Currently this is just the size of the pointer and some estimated bytes.
# The struct isn't actually defined in expat.h - it is in xmlparse.c
# XXX: find a good estimate of the XML_ParserStruct
- rgc.add_memory_pressure(XML_Parser_SIZE + 300)
if not xmlparser:
raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed")
parser = W_XMLParserType(space, xmlparser, w_intern)
+ rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser)
XML_SetUnknownEncodingHandler(
parser.itself, UnknownEncodingHandlerData_callback,
rffi.cast(rffi.VOIDP, parser.id))
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -71,6 +71,7 @@
self.needs_generic_instantiate = {}
self.thread_local_fields = set()
+ self.memory_pressure_types = set()
self.register_builtins()
diff --git a/rpython/jit/codewriter/support.py
b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -675,6 +675,8 @@
def _ll_1_gc_add_memory_pressure(num):
llop.gc_add_memory_pressure(lltype.Void, num)
+ def _ll_2_gc_add_memory_pressure(num, obj):
+ llop.gc_add_memory_pressure(lltype.Void, num, obj)
def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None):
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -83,7 +83,9 @@
has_custom_trace,
fast_path_tracing,
has_gcptr,
- cannot_pin):
+ cannot_pin,
+ has_memory_pressure,
+ get_memory_pressure_ofs):
self.finalizer_handlers = finalizer_handlers
self.destructor_or_custom_trace = destructor_or_custom_trace
self.is_old_style_finalizer = is_old_style_finalizer
@@ -103,6 +105,8 @@
self.fast_path_tracing = fast_path_tracing
self.has_gcptr = has_gcptr
self.cannot_pin = cannot_pin
+ self.has_memory_pressure = has_memory_pressure
+ self.get_memory_pressure_ofs = get_memory_pressure_ofs
def get_member_index(self, type_id):
return self.member_index(type_id)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -72,6 +72,7 @@
from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rgc
from rpython.memory.gc.minimarkpage import out_of_memory
#
@@ -371,6 +372,7 @@
self.old_rawmalloced_objects = self.AddressStack()
self.raw_malloc_might_sweep = self.AddressStack()
self.rawmalloced_total_size = r_uint(0)
+ self.rawmalloced_peak_size = r_uint(0)
self.gc_state = STATE_SCANNING
#
@@ -996,6 +998,8 @@
# Record the newly allocated object and its full malloced size.
# The object is young or old depending on the argument.
self.rawmalloced_total_size += r_uint(allocsize)
+ self.rawmalloced_peak_size = max(self.rawmalloced_total_size,
+ self.rawmalloced_peak_size)
if alloc_young:
if not self.young_rawmalloced_objects:
self.young_rawmalloced_objects = self.AddressDict()
@@ -1023,7 +1027,7 @@
if self.max_heap_size < self.next_major_collection_threshold:
self.next_major_collection_threshold = self.max_heap_size
- def raw_malloc_memory_pressure(self, sizehint):
+ def raw_malloc_memory_pressure(self, sizehint, adr):
# Decrement by 'sizehint' plus a very little bit extra. This
# is needed e.g. for _rawffi, which may allocate a lot of tiny
# arrays.
@@ -1183,6 +1187,24 @@
"""
return self.ac.total_memory_used + self.rawmalloced_total_size
+ def get_total_memory_alloced(self):
+ """ Return the total memory allocated
+ """
+ return self.ac.total_memory_alloced + self.rawmalloced_total_size
+
+ def get_peak_memory_alloced(self):
+ """ Return the peak memory ever allocated. The peaks
+ can be at different times, but we just don't worry for now
+ """
+ return self.ac.peak_memory_alloced + self.rawmalloced_peak_size
+
+ def get_peak_memory_used(self):
+ """ Return the peak memory GC felt ever responsible for
+ """
+ mem_allocated = max(self.ac.peak_memory_used,
+ self.ac.total_memory_used)
+ return mem_allocated + self.rawmalloced_peak_size
+
def threshold_reached(self, extra=0):
return (self.next_major_collection_threshold -
float(self.get_total_memory_used())) < float(extra)
@@ -2155,6 +2177,8 @@
#
size_gc_header = self.gcheaderbuilder.size_gc_header
self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize))
+ self.rawmalloced_peak_size = max(self.rawmalloced_total_size,
+ self.rawmalloced_peak_size)
self.old_rawmalloced_objects.append(arena + size_gc_header)
return arena
@@ -2932,6 +2956,21 @@
self.old_objects_with_weakrefs.delete()
self.old_objects_with_weakrefs = new_with_weakref
+ def get_stats(self, stats_no):
+ from rpython.memory.gc import inspector
+
+ if stats_no == rgc.TOTAL_MEMORY:
+ return intmask(self.get_total_memory_used() + self.nursery_size)
+ elif stats_no == rgc.PEAK_MEMORY:
+ return intmask(self.get_peak_memory_used() + self.nursery_size)
+ elif stats_no == rgc.PEAK_ALLOCATED_MEMORY:
+ return intmask(self.get_peak_memory_alloced() + self.nursery_size)
+ elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY:
+ return intmask(self.get_total_memory_alloced() + self.nursery_size)
+ elif stats_no == rgc.TOTAL_MEMORY_PRESSURE:
+ return inspector.count_memory_pressure(self)
+ return 0
+
# ----------
# RawRefCount
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -2,6 +2,7 @@
Utility RPython functions to inspect objects in the GC.
"""
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
+from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rlib.objectmodel import free_non_gc_object
from rpython.rlib import rposix, rgc, jit
@@ -92,17 +93,12 @@
AddressStack = get_address_stack()
-class HeapDumper(object):
- _alloc_flavor_ = "raw"
- BUFSIZE = 8192 # words
+class BaseWalker(object):
+ _alloc_flavor_ = 'raw'
- def __init__(self, gc, fd):
+ def __init__(self, gc):
self.gc = gc
self.gcflag = gc.gcflag_extra
- self.fd = rffi.cast(rffi.INT, fd)
- self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE,
- flavor='raw')
- self.buf_count = 0
if self.gcflag == 0:
self.seen = AddressDict()
self.pending = AddressStack()
@@ -111,8 +107,107 @@
if self.gcflag == 0:
self.seen.delete()
self.pending.delete()
+ free_non_gc_object(self)
+
+ def add_roots(self):
+ self.gc.enumerate_all_roots(_hd_add_root, self)
+ pendingroots = self.pending
+ self.pending = AddressStack()
+ self.walk(pendingroots)
+ pendingroots.delete()
+ self.end_add_roots_marker()
+
+ def end_add_roots_marker(self):
+ pass
+
+ def add(self, obj):
+ if self.gcflag == 0:
+ if not self.seen.contains(obj):
+ self.seen.setitem(obj, obj)
+ self.pending.append(obj)
+ else:
+ hdr = self.gc.header(obj)
+ if (hdr.tid & self.gcflag) == 0:
+ hdr.tid |= self.gcflag
+ self.pending.append(obj)
+
+ def walk(self, pending):
+ while pending.non_empty():
+ self.processobj(pending.pop())
+
+ # ----------
+ # A simplified copy of the above, to make sure we walk again all the
+ # objects to clear the 'gcflag'.
+
+ def unobj(self, obj):
+ gc = self.gc
+ gc.trace(obj, self._unref, None)
+
+ def _unref(self, pointer, _):
+ obj = pointer.address[0]
+ self.unadd(obj)
+
+ def unadd(self, obj):
+ assert self.gcflag != 0
+ hdr = self.gc.header(obj)
+ if (hdr.tid & self.gcflag) != 0:
+ hdr.tid &= ~self.gcflag
+ self.pending.append(obj)
+
+ def clear_gcflag_again(self):
+ self.gc.enumerate_all_roots(_hd_unadd_root, self)
+ pendingroots = self.pending
+ self.pending = AddressStack()
+ self.unwalk(pendingroots)
+ pendingroots.delete()
+
+ def unwalk(self, pending):
+ while pending.non_empty():
+ self.unobj(pending.pop())
+
+ def finish_processing(self):
+ if self.gcflag != 0:
+ self.clear_gcflag_again()
+ self.unwalk(self.pending)
+
+ def process(self):
+ self.add_roots()
+ self.walk(self.pending)
+
+
+class MemoryPressureCounter(BaseWalker):
+
+ def __init__(self, gc):
+ self.count = 0
+ BaseWalker.__init__(self, gc)
+
+ def processobj(self, obj):
+ gc = self.gc
+ typeid = gc.get_type_id(obj)
+ if gc.has_memory_pressure(typeid):
+ ofs = gc.get_memory_pressure_ofs(typeid)
+ val = (obj + ofs).signed[0]
+ self.count += val
+ gc.trace(obj, self._ref, None)
+
+ def _ref(self, pointer, _):
+ obj = pointer.address[0]
+ self.add(obj)
+
+
+class HeapDumper(BaseWalker):
+ BUFSIZE = 8192 # words
+
+ def __init__(self, gc, fd):
+ BaseWalker.__init__(self, gc)
+ self.fd = rffi.cast(rffi.INT, fd)
+ self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE,
+ flavor='raw')
+ self.buf_count = 0
+
+ def delete(self):
lltype.free(self.writebuffer, flavor='raw')
- free_non_gc_object(self)
+ BaseWalker.delete(self)
@jit.dont_look_inside
def flush(self):
@@ -143,6 +238,7 @@
self.write(0)
self.write(0)
self.write(-1)
+ end_add_roots_marker = write_marker
def writeobj(self, obj):
gc = self.gc
@@ -152,64 +248,13 @@
self.write(gc.get_size_incl_hash(obj))
gc.trace(obj, self._writeref, None)
self.write(-1)
+ processobj = writeobj
def _writeref(self, pointer, _):
obj = pointer.address[0]
self.write(llmemory.cast_adr_to_int(obj))
self.add(obj)
- def add(self, obj):
- if self.gcflag == 0:
- if not self.seen.contains(obj):
- self.seen.setitem(obj, obj)
- self.pending.append(obj)
- else:
- hdr = self.gc.header(obj)
- if (hdr.tid & self.gcflag) == 0:
- hdr.tid |= self.gcflag
- self.pending.append(obj)
-
- def add_roots(self):
- self.gc.enumerate_all_roots(_hd_add_root, self)
- pendingroots = self.pending
- self.pending = AddressStack()
- self.walk(pendingroots)
- pendingroots.delete()
- self.write_marker()
-
- def walk(self, pending):
- while pending.non_empty():
- self.writeobj(pending.pop())
-
- # ----------
- # A simplified copy of the above, to make sure we walk again all the
- # objects to clear the 'gcflag'.
-
- def unwriteobj(self, obj):
- gc = self.gc
- gc.trace(obj, self._unwriteref, None)
-
- def _unwriteref(self, pointer, _):
- obj = pointer.address[0]
- self.unadd(obj)
-
- def unadd(self, obj):
- assert self.gcflag != 0
- hdr = self.gc.header(obj)
- if (hdr.tid & self.gcflag) != 0:
- hdr.tid &= ~self.gcflag
- self.pending.append(obj)
-
- def clear_gcflag_again(self):
- self.gc.enumerate_all_roots(_hd_unadd_root, self)
- pendingroots = self.pending
- self.pending = AddressStack()
- self.unwalk(pendingroots)
- pendingroots.delete()
-
- def unwalk(self, pending):
- while pending.non_empty():
- self.unwriteobj(pending.pop())
def _hd_add_root(obj, heap_dumper):
heap_dumper.add(obj)
@@ -219,15 +264,20 @@
def dump_rpy_heap(gc, fd):
heapdumper = HeapDumper(gc, fd)
- heapdumper.add_roots()
- heapdumper.walk(heapdumper.pending)
+ heapdumper.process()
heapdumper.flush()
- if heapdumper.gcflag != 0:
- heapdumper.clear_gcflag_again()
- heapdumper.unwalk(heapdumper.pending)
+ heapdumper.finish_processing()
heapdumper.delete()
return True
+def count_memory_pressure(gc):
+ counter = MemoryPressureCounter(gc)
+ counter.process()
+ counter.finish_processing()
+ res = counter.count
+ counter.delete()
+ return res
+
def get_typeids_z(gc):
srcaddress = gc.root_walker.gcdata.typeids_z
return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR))
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -828,7 +828,7 @@
if self.max_heap_size < self.next_major_collection_threshold:
self.next_major_collection_threshold = self.max_heap_size
- def raw_malloc_memory_pressure(self, sizehint):
+ def raw_malloc_memory_pressure(self, sizehint, adr):
self.next_major_collection_threshold -= sizehint
if self.next_major_collection_threshold < 0:
# cannot trigger a full collection now, but we can ensure
diff --git a/rpython/memory/gc/minimarkpage.py
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -141,6 +141,9 @@
# the total memory used, counting every block in use, without
# the additional bookkeeping stuff.
self.total_memory_used = r_uint(0)
+ self.peak_memory_used = r_uint(0)
+ self.total_memory_alloced = r_uint(0)
+ self.peak_memory_alloced = r_uint(0)
def _new_page_ptr_list(self, length):
@@ -293,7 +296,11 @@
#
# 'arena_base' points to the start of malloced memory; it might not
# be a page-aligned address
- arena_base = llarena.arena_malloc(self.arena_size, False)
+ arena_base = llarena.arena_mmap(self.arena_size)
+ self.total_memory_alloced += self.arena_size
+ self.peak_memory_alloced = max(self.total_memory_alloced,
+ self.peak_memory_alloced)
+
if not arena_base:
out_of_memory("out of memory: couldn't allocate the next arena")
arena_end = arena_base + self.arena_size
@@ -321,6 +328,8 @@
"""Prepare calls to mass_free_incremental(): moves the chained lists
into 'self.old_xxx'.
"""
+ self.peak_memory_used = max(self.peak_memory_used,
+ self.total_memory_used)
self.total_memory_used = r_uint(0)
#
size_class = self.small_request_threshold >> WORD_POWER_2
@@ -397,8 +406,8 @@
if arena.nfreepages == arena.totalpages:
#
# The whole arena is empty. Free it.
- llarena.arena_reset(arena.base, self.arena_size, 4)
- llarena.arena_free(arena.base)
+ llarena.arena_munmap(arena.base, self.arena_size)
+ self.total_memory_alloced -= self.arena_size
lltype.free(arena, flavor='raw', track_allocation=False)
self.arenas_count -= 1
#
diff --git a/rpython/memory/gctransform/framework.py
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -392,23 +392,30 @@
inline = True)
if getattr(GCClass, 'raw_malloc_memory_pressure', False):
- def raw_malloc_memory_pressure_varsize(length, itemsize):
+ def raw_malloc_memory_pressure_varsize(length, itemsize, adr):
totalmem = length * itemsize
if totalmem > 0:
- gcdata.gc.raw_malloc_memory_pressure(totalmem)
+ gcdata.gc.raw_malloc_memory_pressure(totalmem, adr)
#else: probably an overflow -- the following rawmalloc
# will fail then
- def raw_malloc_memory_pressure(sizehint):
- gcdata.gc.raw_malloc_memory_pressure(sizehint)
+ def raw_malloc_memory_pressure(sizehint, adr):
+ gcdata.gc.raw_malloc_memory_pressure(sizehint, adr)
self.raw_malloc_memory_pressure_varsize_ptr = getfn(
raw_malloc_memory_pressure_varsize,
- [annmodel.SomeInteger(), annmodel.SomeInteger()],
+ [annmodel.SomeInteger(), annmodel.SomeInteger(),
+ SomeAddress()],
annmodel.s_None, minimal_transform = False)
self.raw_malloc_memory_pressure_ptr = getfn(
raw_malloc_memory_pressure,
- [annmodel.SomeInteger()],
+ [annmodel.SomeInteger(), SomeAddress()],
annmodel.s_None, minimal_transform = False)
+ if getattr(GCClass, 'get_stats', False):
+ def get_stats(stats_no):
+ return gcdata.gc.get_stats(stats_no)
+ self.get_stats_ptr = getfn(get_stats, [annmodel.SomeInteger()],
+ annmodel.SomeInteger())
+
self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
[s_gc, s_gcref],
@@ -831,6 +838,39 @@
gct_fv_gc_malloc_varsize = gct_fv_gc_malloc
+ def gct_gc_add_memory_pressure(self, hop):
+ def _find_correct_type(TP):
+ T = TP.TO
+ while 'special_memory_pressure' not in T._flds:
+ T = T._flds['super']
+ return T
+
+ if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
+ op = hop.spaceop
+ size = op.args[0]
+ if len(op.args) == 2:
+ v_fld = rmodel.inputconst(lltype.Void,
"special_memory_pressure")
+ T = _find_correct_type(op.args[1].concretetype)
+ v_inst = hop.genop("cast_pointer", [op.args[1]],
+ resulttype=lltype.Ptr(T))
+ hop.genop("bare_setfield", [v_inst, v_fld, size])
+ v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
+ resulttype=llmemory.Address)
+ else:
+ v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
+ hop.genop("direct_call", [self.raw_malloc_memory_pressure_ptr,
+ size, v_adr])
+
+
+ def gct_gc_get_stats(self, hop):
+ if hasattr(self, 'get_stats_ptr'):
+ return hop.genop("direct_call",
+ [self.get_stats_ptr, hop.spaceop.args[0]],
+ resultvar=hop.spaceop.result)
+ hop.genop("same_as", [rmodel.inputconst(lltype.Signed, 0)],
+ resultvar=hop.spaceop.result)
+
+
def gct_gc__collect(self, hop):
op = hop.spaceop
if len(op.args) == 1:
diff --git a/rpython/memory/gctransform/transform.py
b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -535,12 +535,7 @@
return self.varsize_malloc_helper(hop, flags, meth, [])
def gct_gc_add_memory_pressure(self, hop):
- if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
- op = hop.spaceop
- size = op.args[0]
- return hop.genop("direct_call",
- [self.raw_malloc_memory_pressure_ptr,
- size])
+ pass
def varsize_malloc_helper(self, hop, flags, meth, extraargs):
def intconst(c): return rmodel.inputconst(lltype.Signed, c)
@@ -574,9 +569,10 @@
c_offset_to_length):
if flags.get('add_memory_pressure', False):
if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'):
+ v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
hop.genop("direct_call",
[self.raw_malloc_memory_pressure_varsize_ptr,
- v_length, c_item_size])
+ v_length, c_item_size, v_adr])
if c_offset_to_length is None:
if flags.get('zero'):
fnptr = self.raw_malloc_varsize_no_length_zero_ptr
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -21,13 +21,21 @@
# A destructor is called when the object is about to be freed.
# A custom tracer (CT) enumerates the addresses that contain GCREFs.
# Both are called with the address of the object as only argument.
+ # They're embedded in a struct that has raw_memory_offset as another
+ # argument, which is only valid if T_HAS_MEMORY_PRESSURE is set
CUSTOM_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
CUSTOM_FUNC_PTR = lltype.Ptr(CUSTOM_FUNC)
+ CUSTOM_DATA_STRUCT = lltype.Struct('custom_data',
+ ('customfunc', CUSTOM_FUNC_PTR),
+ ('memory_pressure_offset', lltype.Signed), # offset to where the amount
+ # of owned memory pressure is stored
+ )
+ CUSTOM_DATA_STRUCT_PTR = lltype.Ptr(CUSTOM_DATA_STRUCT)
# structure describing the layout of a typeid
TYPE_INFO = lltype.Struct("type_info",
("infobits", lltype.Signed), # combination of the T_xxx consts
- ("customfunc", CUSTOM_FUNC_PTR),
+ ("customdata", CUSTOM_DATA_STRUCT_PTR),
("fixedsize", lltype.Signed),
("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)),
hints={'immutable': True},
@@ -81,14 +89,14 @@
def q_cannot_pin(self, typeid):
typeinfo = self.get(typeid)
ANY = (T_HAS_GCPTR | T_IS_WEAKREF)
- return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customfunc)
+ return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customdata)
def q_finalizer_handlers(self):
adr = self.finalizer_handlers # set from framework.py or gcwrapper.py
return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY))
def q_destructor_or_custom_trace(self, typeid):
- return self.get(typeid).customfunc
+ return self.get(typeid).customdata.customfunc
def q_is_old_style_finalizer(self, typeid):
typeinfo = self.get(typeid)
@@ -139,6 +147,15 @@
infobits = self.get(typeid).infobits
return infobits & T_ANY_SLOW_FLAG == 0
+ def q_has_memory_pressure(self, typeid):
+ infobits = self.get(typeid).infobits
+ return infobits & T_HAS_MEMORY_PRESSURE != 0
+
+ def q_get_memory_pressure_ofs(self, typeid):
+ infobits = self.get(typeid).infobits
+ assert infobits & T_HAS_MEMORY_PRESSURE != 0
+ return self.get(typeid).customdata.memory_pressure_offset
+
def set_query_functions(self, gc):
gc.set_query_functions(
self.q_is_varsize,
@@ -159,7 +176,9 @@
self.q_has_custom_trace,
self.q_fast_path_tracing,
self.q_has_gcptr,
- self.q_cannot_pin)
+ self.q_cannot_pin,
+ self.q_has_memory_pressure,
+ self.q_get_memory_pressure_ofs)
def _has_got_custom_trace(self, typeid):
type_info = self.get(typeid)
@@ -176,8 +195,9 @@
T_HAS_CUSTOM_TRACE = 0x200000
T_HAS_OLDSTYLE_FINALIZER = 0x400000
T_HAS_GCPTR = 0x1000000
-T_KEY_MASK = intmask(0xFE000000) # bug detection only
-T_KEY_VALUE = intmask(0x5A000000) # bug detection only
+T_HAS_MEMORY_PRESSURE = 0x2000000 # first field is memory pressure field
+T_KEY_MASK = intmask(0xFC000000) # bug detection only
+T_KEY_VALUE = intmask(0x58000000) # bug detection only
def _check_valid_type_info(p):
ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -192,6 +212,25 @@
ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid),
"invalid type_id")
+def has_special_memory_pressure(TYPE):
+ if TYPE._is_varsize():
+ return False
+ T = TYPE
+ while True:
+ if 'special_memory_pressure' in T._flds:
+ return True
+ if 'super' not in T._flds:
+ return False
+ T = T._flds['super']
+
+def get_memory_pressure_ofs(TYPE):
+ T = TYPE
+ while True:
+ if 'special_memory_pressure' in T._flds:
+ return llmemory.offsetof(T, 'special_memory_pressure')
+ if 'super' not in T._flds:
+ assert False, "get_ and has_memory_pressure disagree"
+ T = T._flds['super']
def encode_type_shape(builder, info, TYPE, index):
"""Encode the shape of the TYPE into the TYPE_INFO structure 'info'."""
@@ -202,12 +241,18 @@
infobits |= T_HAS_GCPTR
#
fptrs = builder.special_funcptr_for_type(TYPE)
- if fptrs:
+ if fptrs or has_special_memory_pressure(TYPE):
+ customdata = lltype.malloc(GCData.CUSTOM_DATA_STRUCT, flavor='raw',
+ immortal=True)
+ info.customdata = customdata
if "destructor" in fptrs:
- info.customfunc = fptrs["destructor"]
+ customdata.customfunc = fptrs["destructor"]
if "old_style_finalizer" in fptrs:
- info.customfunc = fptrs["old_style_finalizer"]
+ customdata.customfunc = fptrs["old_style_finalizer"]
infobits |= T_HAS_OLDSTYLE_FINALIZER
+ if has_special_memory_pressure(TYPE):
+ infobits |= T_HAS_MEMORY_PRESSURE
+ info.customdata.memory_pressure_offset =
get_memory_pressure_ofs(TYPE)
#
if not TYPE._is_varsize():
info.fixedsize = llarena.round_up_for_allocation(
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -83,9 +83,9 @@
def gettypeid(self, obj):
return self.get_type_id(lltype.typeOf(obj).TO)
- def add_memory_pressure(self, size):
+ def add_memory_pressure(self, size, adr):
if hasattr(self.gc, 'raw_malloc_memory_pressure'):
- self.gc.raw_malloc_memory_pressure(size)
+ self.gc.raw_malloc_memory_pressure(size, adr)
def shrink_array(self, p, smallersize):
if hasattr(self.gc, 'shrink_array'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -598,21 +598,31 @@
return False
return type(x).__module__ != '__builtin__' # keep non-builtins
-def add_memory_pressure(estimate):
+def add_memory_pressure(estimate, object=None):
"""Add memory pressure for OpaquePtrs."""
pass
class AddMemoryPressureEntry(ExtRegistryEntry):
_about_ = add_memory_pressure
- def compute_result_annotation(self, s_nbytes):
+ def compute_result_annotation(self, s_nbytes, s_object=None):
from rpython.annotator import model as annmodel
+ if s_object is not None:
+ if not isinstance(s_object, annmodel.SomeInstance):
+ raise Exception("Wrong kind of object passed to "
+ "add memory pressure")
+ self.bookkeeper.memory_pressure_types.add(s_object.classdef)
return annmodel.s_None
def specialize_call(self, hop):
- [v_size] = hop.inputargs(lltype.Signed)
+ v_size = hop.inputarg(lltype.Signed, 0)
+ if len(hop.args_v) == 2:
+ v_obj = hop.inputarg(hop.args_r[1], 1)
+ args = [v_size, v_obj]
+ else:
+ args = [v_size]
hop.exception_cannot_occur()
- return hop.genop('gc_add_memory_pressure', [v_size],
+ return hop.genop('gc_add_memory_pressure', args,
resulttype=lltype.Void)
@@ -640,6 +650,15 @@
else:
return id(gcref._x)
+(TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE,
+ PEAK_MEMORY, PEAK_ALLOCATED_MEMORY) = range(5)
+
+@not_rpython
+def get_stats(stat_no):
+ """ Long docstring goes here
+ """
+ raise NotImplementedError
+
@not_rpython
def dump_rpy_heap(fd):
raise NotImplementedError
@@ -834,6 +853,18 @@
return hop.genop('gc_get_rpy_type_index', vlist,
resulttype = hop.r_result)
+class Entry(ExtRegistryEntry):
+ _about_ = get_stats
+ def compute_result_annotation(self, s_no):
+ from rpython.annotator.model import SomeInteger
+ if not isinstance(s_no, SomeInteger):
+ raise Exception("expecting an integer")
+ return SomeInteger()
+ def specialize_call(self, hop):
+ args = hop.inputargs(lltype.Signed)
+ hop.exception_cannot_occur()
+ return hop.genop('gc_get_stats', args, resulttype=lltype.Signed)
+
@not_rpython
def _is_rpy_instance(gcref):
raise NotImplementedError
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -85,7 +85,11 @@
def allocate_lock():
- return Lock(allocate_ll_lock())
+ # Add some memory pressure for the size of the lock because it is an
+ # Opaque object
+ lock = Lock(allocate_ll_lock())
+ rgc.add_memory_pressure(TLOCKP_SIZE, lock)
+ return lock
@specialize.arg(0)
def ll_start_new_thread(func):
@@ -248,9 +252,6 @@
if rffi.cast(lltype.Signed, res) <= 0:
lltype.free(ll_lock, flavor='raw', track_allocation=False)
raise error("out of resources")
- # Add some memory pressure for the size of the lock because it is an
- # Opaque object
- rgc.add_memory_pressure(TLOCKP_SIZE)
return ll_lock
def free_ll_lock(ll_lock):
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -269,7 +269,6 @@
compress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
- rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
if err == Z_OK:
if zdict is not None:
@@ -304,7 +303,6 @@
decompress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
- rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _inflateInit2(stream, wbits)
if err == Z_OK:
if zdict is not None and wbits < 0:
diff --git a/rpython/rtyper/lltypesystem/llarena.py
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -327,6 +327,17 @@
assert not arena_addr.arena.objectptrs
arena_addr.arena.mark_freed()
+def arena_mmap(nbytes):
+ """Allocate and return a new arena, zero-initialized by the
+ system, calling mmap()."""
+ return arena_malloc(nbytes, True)
+
+def arena_munmap(arena_addr, nbytes):
+ """Release an arena allocated with arena_mmap()."""
+ arena_free(arena_addr)
+ assert nbytes == arena_addr.arena.nbytes
+
+
def arena_reset(arena_addr, size, zero):
"""Free all objects in the arena, which can then be reused.
This can also be used on a subrange of the arena.
@@ -530,6 +541,31 @@
llfakeimpl=arena_free,
sandboxsafe=True)
+def llimpl_arena_mmap(nbytes):
+ from rpython.rlib import rmmap
+ flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS
+ prot = rmmap.PROT_READ | rmmap.PROT_WRITE
+ p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe(
+ lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0))
+ if p == rffi.cast(llmemory.Address, -1):
+ p = rffi.cast(llmemory.Address, 0)
+ return p
+register_external(arena_mmap, [int], llmemory.Address,
+ 'll_arena.arena_mmap',
+ llimpl=llimpl_arena_mmap,
+ llfakeimpl=arena_mmap,
+ sandboxsafe=True)
+
+def llimpl_arena_munmap(arena_addr, nbytes):
+ from rpython.rlib import rmmap
+ assert nbytes >= 0
+ rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes)
+register_external(arena_munmap, [llmemory.Address, int], None,
+ 'll_arena.arena_munmap',
+ llimpl=llimpl_arena_munmap,
+ llfakeimpl=arena_munmap,
+ sandboxsafe=True)
+
def llimpl_arena_reset(arena_addr, size, zero):
if zero:
if zero == 1:
diff --git a/rpython/rtyper/lltypesystem/lloperation.py
b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -485,6 +485,7 @@
'gc_gettypeid' : LLOp(),
'gc_gcflag_extra' : LLOp(),
'gc_add_memory_pressure': LLOp(),
+ 'gc_get_stats' : LLOp(),
'gc_fq_next_dead' : LLOp(),
'gc_fq_register' : LLOp(),
'gc_ignore_finalizer' : LLOp(canrun=True),
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -15,6 +15,7 @@
RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, Void, FuncType, Bool, Signed,
functionptr, attachRuntimeTypeInfo)
from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.llannotation import lltype_to_annotation
from rpython.rtyper.llannotation import SomePtr
from rpython.rtyper.lltypesystem import rstr
from rpython.rtyper.rmodel import (
@@ -475,6 +476,13 @@
self.lowleveltype = Ptr(self.object_type)
self.gcflavor = gcflavor
+ def has_special_memory_pressure(self, tp):
+ if 'special_memory_pressure' in tp._flds:
+ return True
+ if 'super' in tp._flds:
+ return self.has_special_memory_pressure(tp._flds['super'])
+ return False
+
def _setup_repr(self, llfields=None, hints=None, adtmeths=None):
# NOTE: don't store mutable objects like the dicts below on 'self'
# before they are fully built, to avoid strange bugs in case
@@ -523,6 +531,16 @@
if not attrdef.readonly and self.is_quasi_immutable(name):
llfields.append(('mutate_' + name, OBJECTPTR))
+ bookkeeper = self.rtyper.annotator.bookkeeper
+ if self.classdef in bookkeeper.memory_pressure_types:
+ # we don't need to add it if it's already there for some of
+ # the parent type
+ if not
self.has_special_memory_pressure(self.rbase.object_type):
+ llfields.append(('special_memory_pressure', lltype.Signed))
+ fields['special_memory_pressure'] = (
+ 'special_memory_pressure',
+
self.rtyper.getrepr(lltype_to_annotation(lltype.Signed)))
+
object_type = MkStruct(self.classdef.name,
('super', self.rbase.object_type),
hints=hints,
@@ -663,6 +681,8 @@
while base.classdef is not None:
base = base.rbase
for fieldname in base.fields:
+ if fieldname == 'special_memory_pressure':
+ continue
try:
mangled, r = base._get_field(fieldname)
except KeyError:
@@ -717,6 +737,9 @@
resulttype=Ptr(self.object_type))
ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
self.setfield(vptr, '__class__', ctypeptr, llops)
+ if self.has_special_memory_pressure(self.object_type):
+ self.setfield(vptr, 'special_memory_pressure',
+ inputconst(lltype.Signed, 0), llops)
# initialize instance attributes from their defaults from the class
if self.classdef is not None:
flds = self.allinstancefields.keys()
diff --git a/rpython/translator/backendopt/writeanalyze.py
b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -65,6 +65,11 @@
elif op.opname == "gc_store_indexed":
if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]):
return self._gc_store_indexed_result(op)
+ elif op.opname == 'gc_add_memory_pressure':
+ # special_memory_pressure would be overwritten by zero, because
+ # the JIT cannot see the field write, which is why we assume
+ # it can write anything
+ return top_set
return empty_set
def _array_result(self, TYPE):
diff --git a/rpython/translator/c/test/test_newgc.py
b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1613,7 +1613,7 @@
digest = ropenssl.EVP_get_digestbyname('sha1')
self.ctx = ropenssl.EVP_MD_CTX_new()
ropenssl.EVP_DigestInit(self.ctx, digest)
- rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
+ rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64, self)
def __del__(self):
ropenssl.EVP_MD_CTX_free(self.ctx)
@@ -1624,12 +1624,16 @@
am3 = am2
am2 = am1
am1 = A()
+ am1 = am2 = am3 = None
# what can we use for the res?
- return 0
+ for i in range(10):
+ gc.collect()
+ return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
return f
def test_nongc_opaque_attached_to_gc(self):
res = self.run("nongc_opaque_attached_to_gc")
+ # the res is 0 for non-memory-pressure-accounting GC
assert res == 0
def define_limited_memory(self):
@@ -1668,6 +1672,38 @@
class TestIncrementalMiniMarkGC(TestMiniMarkGC):
gcpolicy = "incminimark"
+ def define_total_memory_pressure(cls):
+ class A(object):
+ def __init__(self):
+ rgc.add_memory_pressure(10, self)
+
+ def __del__(self):
+ pass
+
+ class B(A):
+ def __init__(self):
+ rgc.add_memory_pressure(10, self)
+
+ class C(A):
+ pass
+
+ class Glob(object):
+ pass
+ glob = Glob()
+
+ def f():
+ glob.l = [None] * 3
+ for i in range(10000):
+ glob.l[i % 3] = A()
+ glob.l[(i + 1) % 3] = B()
+ glob.l[(i + 2) % 3] = C()
+ return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+ return f
+
+ def test_total_memory_pressure(self):
+ res = self.run("total_memory_pressure")
+ assert res == 30 # total reachable is 3
+
def define_random_pin(self):
class A:
foo = None
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
