Author: fijal
Branch: memory-accounting
Changeset: r93011:5e198814c5f6
Date: 2017-11-14 10:19 +0100
http://bitbucket.org/pypy/pypy/changeset/5e198814c5f6/
Log: add tracking of memory
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -19,6 +19,7 @@
space.config.translation.gctransformer == "framework"):
self.appleveldefs.update({
'dump_rpy_heap': 'app_referents.dump_rpy_heap',
+ 'get_stats': 'app_referents.get_stats',
})
self.interpleveldefs.update({
'get_rpy_roots': 'referents.get_rpy_roots',
@@ -28,7 +29,7 @@
'get_objects': 'referents.get_objects',
'get_referents': 'referents.get_referents',
'get_referrers': 'referents.get_referrers',
- 'get_stats': 'referents.get_stats',
+ '_get_stats': 'referents.get_stats',
'_dump_rpy_heap': 'referents._dump_rpy_heap',
'get_typeids_z': 'referents.get_typeids_z',
'get_typeids_list': 'referents.get_typeids_list',
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -48,3 +48,42 @@
file.flush()
fd = file.fileno()
gc._dump_rpy_heap(fd)
+
+class GcStats(object):
+ def __init__(self, s):
+ self._s = s
+ for item in ('total_gc_memory', 'jit_backend_used',
'total_memory_pressure',
+ 'total_allocated_memory', 'jit_backend_allocated'):
+ setattr(self, item, self._format(getattr(self._s, item)))
+ self.memory_used_sum = self._format(self._s.total_gc_memory +
self._s.total_memory_pressure +
+ self._s.jit_backend_used)
+ self.memory_allocated_sum =
self._format(self._s.total_allocated_memory + self._s.total_memory_pressure +
+ self._s.jit_backend_allocated)
+
+ def _format(self, v):
+ if v < 1000000:
+ # bit unlikely ;-)
+ return "%.1fkB" % (v / 1024.)
+ return "%.1fMB" % (v / 1024. / 1024.)
+
+ def repr(self):
+ return """Total memory consumed:
+GC used: %s
+raw assembler used: %s
+memory pressure: %s
+-----------------------------
+Total: %s
+
+Total memory allocated:
+GC allocated: %s
+raw assembler allocated: %s
+memory pressure: %s
+-----------------------------
+Total: %s
+""" % (self.total_gc_memory, self.jit_backend_used, self.total_memory_pressure,
+ self.memory_used_sum,
+ self.total_allocated_memory, self.jit_backend_allocated,
self.total_memory_pressure,
+ self.memory_allocated_sum)
+
+def get_stats():
+ return GcStats(gc._get_stats())
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -1,7 +1,7 @@
-from rpython.rlib import rgc
+from rpython.rlib import rgc, jit_hooks
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.typedef import TypeDef, interp_attrproperty
-from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.gateway import unwrap_spec, interp2app
from pypy.interpreter.error import oefmt, wrap_oserror
from rpython.rlib.objectmodel import we_are_translated
@@ -175,12 +175,21 @@
def __init__(self):
self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY)
+ self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY)
+ self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None)
+ self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None)
W_GcStats.typedef = TypeDef("GcStats",
total_memory_pressure=interp_attrproperty("total_memory_pressure",
cls=W_GcStats, wrapfn="newint"),
total_gc_memory=interp_attrproperty("total_gc_memory",
- cls=W_GcStats, wrapfn="newint")
+ cls=W_GcStats, wrapfn="newint"),
+ total_allocated_memory=interp_attrproperty("total_allocated_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ jit_backend_allocated=interp_attrproperty("jit_backend_allocated",
+ cls=W_GcStats, wrapfn="newint"),
+ jit_backend_used=interp_attrproperty("jit_backend_used",
+ cls=W_GcStats, wrapfn="newint"),
)
def get_stats(space):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1184,6 +1184,11 @@
"""
return self.ac.total_memory_used + self.rawmalloced_total_size
+ def get_total_memory_alloced(self):
+ """ Return the total memory allocated
+ """
+ return self.ac.total_memory_alloced + self.rawmalloced_total_size
+
def threshold_reached(self, extra=0):
return (self.next_major_collection_threshold -
float(self.get_total_memory_used())) < float(extra)
@@ -2925,7 +2930,7 @@
if stats_no == rgc.TOTAL_MEMORY:
return intmask(self.get_total_memory_used() + self.nursery_size)
elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY:
- return 0
+ return intmask(self.get_total_memory_alloced() + self.nursery_size)
elif stats_no == rgc.TOTAL_MEMORY_PRESSURE:
return inspector.count_memory_pressure(self)
return 0
diff --git a/rpython/memory/gc/minimarkpage.py
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -294,6 +294,7 @@
# 'arena_base' points to the start of malloced memory; it might not
# be a page-aligned address
arena_base = llarena.arena_malloc(self.arena_size, False)
+ self.total_memory_alloced += self.arena_size
if not arena_base:
out_of_memory("out of memory: couldn't allocate the next arena")
arena_end = arena_base + self.arena_size
@@ -398,6 +399,7 @@
# The whole arena is empty. Free it.
llarena.arena_reset(arena.base, self.arena_size, 4)
llarena.arena_free(arena.base)
+ self.total_memory_alloced -= self.arena_size
lltype.free(arena, flavor='raw', track_allocation=False)
#
else:
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
