Author: Armin Rigo <[email protected]>
Branch: gc-del
Changeset: r62010:84eb06294ac8
Date: 2013-03-04 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/84eb06294ac8/
Log: More in-progress.
diff --git a/pypy/doc/discussion/finalizer-order.rst
b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -39,7 +39,7 @@
by the GC when the last reference to the object goes away, like in
CPython. However (like "lightweight finalizers" used to be), all
``__del__()`` methods must only contain simple enough code, and this
- is checked.
+ is checked. This is now called "destructors".
* For any more advanced usage --- in particular for any app-level object
with a __del__ --- we don't use the RPython-level ``__del__()``
@@ -54,13 +54,12 @@
last; nothing can be done with the object any more.
-Lightweight finalizers
-----------------------
+Destructors
+-----------
-A lightweight finalizer is an RPython ``__del__()`` method that is
-called directly by the GC when there is no more reference to an object
-(including from other objects with finalizers). Intended for objects
-that just need to free a block of raw memory or close a file.
+A destructor is an RPython ``__del__()`` method that is called directly
+by the GC when there is no more reference to an object. Intended for
+objects that just need to free a block of raw memory or close a file.
There are restrictions on the kind of code you can put in ``__del__()``,
including all other functions called by it. These restrictions are
@@ -68,11 +67,16 @@
and if you call an external C function, it must be a "safe" function.
(XXX check/implement this)
+If there are several objects with destructors, they are called in a
+random order --- but that should be fine because destructors cannot
+do much anyway.
+
Register_finalizer
------------------
-The interface is made with PyPy in mind, but should be generally useful.
+The interface for full finalizers is made with PyPy in mind, but should
+be generally useful.
``rgc.register_finalizer(obj.finalizer_method)``
@@ -82,9 +86,9 @@
picks a topological ordering (breaking cycles randomly) and enqueues
the objects and their registered finalizer functions in that order.
Finally, when the major collection is done, it calls
- ``rgc.progress_through_finalizer_queue()`` once, unless there is
+ ``rgc.progress_through_finalizer_queue()`` once (unless there is
already a call to ``rgc.progress_through_finalizer_queue()`` in
- progress.
+ progress).
It is only allowed to register one finalizer per object,
but we can cumulate one register_finalizer() and a __del__(). It is
@@ -99,15 +103,16 @@
function remains at the front of the queue, and will be called again
by the next call to ``progress_through_finalizer_queue()``.
-The idea is that the finalizer functions in PyPy either do their clean-up
-immediately (for the case where they are not lightweight finalizers, but
-don't require synchronization), or are postponed to be executed at the
-end of the current bytecode by the interpreter. This is done by writing
-such functions with this logic::
+The idea is that the finalizer functions in PyPy either do their
+clean-up immediately (for the case where destructors are not
+appropriate, but they still don't require complete synchronization with
+the Python code), or are postponed to be executed at the end of the
+current bytecode by the interpreter. This is done by writing such
+functions with this logic::
def finalize(self):
ec = self.space.getexecutioncontext()
- if not ec.running_finalizers:
+ if not ec.running_finalizers_between_bytecodes:
ec.schedule_later_call_to_progress_through_finalizer_queue()
raise FinalizeLater
else:
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -62,8 +62,7 @@
def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
is_gcarrayofgcptr,
- getfinalizer,
- getlightfinalizer,
+ getdestructor,
offsets_to_gc_pointers,
fixed_size, varsize_item_sizes,
varsize_offset_to_variable_part,
@@ -75,8 +74,7 @@
has_custom_trace,
get_custom_trace,
fast_path_tracing):
- self.getfinalizer = getfinalizer
- self.getlightfinalizer = getlightfinalizer
+ self.getdestructor = getdestructor
self.is_varsize = is_varsize
self.has_gcptr_in_varsize = has_gcptr_in_varsize
self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -138,10 +136,9 @@
# to malloc_varsize(), but always uses malloc_varsize_clear()
size = self.fixed_size(typeid)
- needs_finalizer = bool(self.getfinalizer(typeid))
- finalizer_is_light = bool(self.getlightfinalizer(typeid))
+ has_destructor = bool(self.getdestructor(typeid))
contains_weakptr = self.weakpointer_offset(typeid) >= 0
- assert not (needs_finalizer and contains_weakptr)
+ assert not (has_destructor and contains_weakptr)
if self.is_varsize(typeid):
assert not contains_weakptr
assert not needs_finalizer
@@ -158,8 +155,7 @@
malloc_fixedsize = self.malloc_fixedsize_clear
else:
malloc_fixedsize = self.malloc_fixedsize
- ref = malloc_fixedsize(typeid, size, needs_finalizer,
- finalizer_is_light,
+ ref = malloc_fixedsize(typeid, size, has_destructor,
contains_weakptr)
# lots of cast and reverse-cast around...
return llmemory.cast_ptr_to_adr(ref)
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -104,9 +104,8 @@
# translation and is statically recorded.
GCFLAG_HAS_SHADOW = first_gcflag << 3
-# The following flag is set temporarily on some objects during a major
-# collection. See pypy/doc/discussion/finalizer-order.txt
-GCFLAG_FINALIZATION_ORDERING = first_gcflag << 4
+# The following flag is set when we call rgc.register_finalizer().
+GCFLAG_HAS_FINALIZER= first_gcflag << 4
# This flag is reserved for RPython.
GCFLAG_EXTRA = first_gcflag << 5
@@ -294,10 +293,9 @@
self.old_rawmalloced_objects = self.AddressStack()
self.rawmalloced_total_size = r_uint(0)
#
- # A list of all objects with finalizers (these are never young).
- self.objects_with_finalizers = self.AddressDeque()
- self.young_objects_with_light_finalizers = self.AddressStack()
- self.old_objects_with_light_finalizers = self.AddressStack()
+ # Two lists of all objects with destructors
+ self.young_objects_with_destructors = self.AddressStack()
+ self.old_objects_with_destructors = self.AddressStack()
#
# Two lists of the objects with weakrefs. No weakref can be an
# old object weakly pointing to a young object: indeed, weakrefs
@@ -467,25 +465,16 @@
def malloc_fixedsize_clear(self, typeid, size,
- needs_finalizer=False,
- is_finalizer_light=False,
+ has_destructor=False,
contains_weakptr=False):
size_gc_header = self.gcheaderbuilder.size_gc_header
totalsize = size_gc_header + size
rawtotalsize = raw_malloc_usage(totalsize)
#
- # If the object needs a finalizer, ask for a rawmalloc.
- # The following check should be constant-folded.
- if needs_finalizer and not is_finalizer_light:
- ll_assert(not contains_weakptr,
- "'needs_finalizer' and 'contains_weakptr' both specified")
- obj = self.external_malloc(typeid, 0, can_make_young=False)
- self.objects_with_finalizers.append(obj)
- #
# If totalsize is greater than nonlarge_max (which should never be
# the case in practice), ask for a rawmalloc. The following check
# should be constant-folded.
- elif rawtotalsize > self.nonlarge_max:
+ if rawtotalsize > self.nonlarge_max:
ll_assert(not contains_weakptr,
"'contains_weakptr' specified for a large object")
obj = self.external_malloc(typeid, 0)
@@ -507,11 +496,12 @@
# Build the object.
llarena.arena_reserve(result, totalsize)
obj = result + size_gc_header
- if is_finalizer_light:
- self.young_objects_with_light_finalizers.append(obj)
self.init_gc_object(result, typeid, flags=0)
#
- # If it is a weakref, record it (check constant-folded).
+ # If it is a weakref or has a destructor, record it
+ # (checks constant-folded).
+ if has_destructor:
+ self.young_objects_with_destructors.append(obj)
if contains_weakptr:
self.young_objects_with_weakrefs.append(obj)
#
@@ -876,7 +866,7 @@
"""
assert self.is_in_nursery(obj)
tid = self.header(obj).tid
- result = (tid & GCFLAG_FINALIZATION_ORDERING != 0)
+ result = (tid & GCFLAG_NO_HEAP_PTRS != 0)
if result:
ll_assert(tid == -42, "bogus header for young obj")
else:
@@ -929,9 +919,6 @@
# the GCFLAG_VISITED should not be set between collections
ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
"unexpected GCFLAG_VISITED")
- # the GCFLAG_FINALIZATION_ORDERING should not be set between coll.
- ll_assert(self.header(obj).tid & GCFLAG_FINALIZATION_ORDERING == 0,
- "unexpected GCFLAG_FINALIZATION_ORDERING")
# the GCFLAG_CARDS_SET should not be set between collections
ll_assert(self.header(obj).tid & GCFLAG_CARDS_SET == 0,
"unexpected GCFLAG_CARDS_SET")
@@ -1251,8 +1238,8 @@
# weakrefs' targets.
if self.young_objects_with_weakrefs.non_empty():
self.invalidate_young_weakrefs()
- if self.young_objects_with_light_finalizers.non_empty():
- self.deal_with_young_objects_with_finalizers()
+ if self.young_objects_with_destructors.non_empty():
+ self.deal_with_young_objects_with_destructors()
#
# Clear this mapping.
if self.nursery_objects_shadows.length() > 0:
@@ -1569,16 +1556,18 @@
# with a finalizer and all objects reachable from there (and also
# moves some objects from 'objects_with_finalizers' to
# 'run_finalizers').
- if self.objects_with_finalizers.non_empty():
- self.deal_with_objects_with_finalizers()
+ #if self.objects_with_finalizers.non_empty():
+ # self.deal_with_objects_with_finalizers()
#
self.objects_to_trace.delete()
#
# Weakref support: clear the weak pointers to dying objects
if self.old_objects_with_weakrefs.non_empty():
self.invalidate_old_weakrefs()
- if self.old_objects_with_light_finalizers.non_empty():
- self.deal_with_old_objects_with_finalizers()
+ #
+ # Destructor support
+ if self.old_objects_with_destructors.non_empty():
+ self.deal_with_old_objects_with_destructors()
#
# Walk all rawmalloced objects and free the ones that don't
@@ -1834,24 +1823,23 @@
self.extra_threshold += diff
- # ----------
- # Finalizers
+ # -----------
+ # Destructors
- def deal_with_young_objects_with_finalizers(self):
- """ This is a much simpler version of dealing with finalizers
- and an optimization - we can reasonably assume that those finalizers
- don't do anything fancy and *just* call them. Among other things
- they won't resurrect objects
+ def deal_with_young_objects_with_destructors(self):
+ """We can reasonably assume that destructors don't do anything
+ fancy and *just* call them. Among other things they won't
+ resurrect objects
"""
- while self.young_objects_with_light_finalizers.non_empty():
- obj = self.young_objects_with_light_finalizers.pop()
+ while self.young_objects_with_destructors.non_empty():
+ obj = self.young_objects_with_destructors.pop()
if not self.is_forwarded(obj):
- finalizer = self.getlightfinalizer(self.get_type_id(obj))
- ll_assert(bool(finalizer), "no light finalizer found")
- finalizer(obj, llmemory.NULL)
+ destructor = self.getdestructor(self.get_type_id(obj))
+ ll_assert(bool(destructor), "destructor missing")
+ destructor(obj, llmemory.NULL)
else:
obj = self.get_forwarding_address(obj)
- self.old_objects_with_light_finalizers.append(obj)
+ self.old_objects_with_destructors.append(obj)
def deal_with_old_objects_with_finalizers(self):
""" This is a much simpler version of dealing with finalizers
@@ -1874,6 +1862,7 @@
self.old_objects_with_light_finalizers = new_objects
def deal_with_objects_with_finalizers(self):
+ XXXXXXXXX
# Walk over list of objects with finalizers.
# If it is not surviving, add it to the list of to-be-called
# finalizers and make it survive, to make the finalizer runnable.
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -22,15 +22,15 @@
# custom tracer (CT), it enumerates the addresses that contain GCREFs.
# It is called with the object as first argument, and the previous
# returned address (or NULL the first time) as the second argument.
- FINALIZER_OR_CT_FUNC = lltype.FuncType([llmemory.Address,
+ DESTRUCTOR_OR_CT_FUNC = lltype.FuncType([llmemory.Address,
llmemory.Address],
llmemory.Address)
- FINALIZER_OR_CT = lltype.Ptr(FINALIZER_OR_CT_FUNC)
+ DESTRUCTOR_OR_CT = lltype.Ptr(DESTRUCTOR_OR_CT_FUNC)
# structure describing the layout of a typeid
TYPE_INFO = lltype.Struct("type_info",
("infobits", lltype.Signed), # combination of the T_xxx consts
- ("finalizer_or_customtrace", FINALIZER_OR_CT),
+ ("destructor_or_customtrace", DESTRUCTOR_OR_CT),
("fixedsize", lltype.Signed),
("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)),
hints={'immutable': True},
@@ -77,19 +77,12 @@
infobits = self.get(typeid).infobits
return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0
- def q_finalizer(self, typeid):
+ def q_getdestructor(self, typeid):
typeinfo = self.get(typeid)
- if typeinfo.infobits & T_HAS_FINALIZER:
- return typeinfo.finalizer_or_customtrace
+ if typeinfo.infobits & T_HAS_DESTRUCTOR:
+ return typeinfo.destructor_or_customtrace
else:
- return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
-
- def q_light_finalizer(self, typeid):
- typeinfo = self.get(typeid)
- if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
- return typeinfo.finalizer_or_customtrace
- else:
- return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
+ return lltype.nullptr(GCData.DESTRUCTOR_OR_CT_FUNC)
def q_offsets_to_gc_pointers(self, typeid):
return self.get(typeid).ofstoptrs
@@ -131,7 +124,7 @@
ll_assert(self.q_has_custom_trace(typeid),
"T_HAS_CUSTOM_TRACE missing")
typeinfo = self.get(typeid)
- return typeinfo.finalizer_or_customtrace
+ return typeinfo.destructor_or_customtrace
def q_fast_path_tracing(self, typeid):
# return True if none of the flags T_HAS_GCPTR_IN_VARSIZE,
@@ -147,8 +140,7 @@
self.q_is_varsize,
self.q_has_gcptr_in_varsize,
self.q_is_gcarrayofgcptr,
- self.q_finalizer,
- self.q_light_finalizer,
+ self.q_getdestructor,
self.q_offsets_to_gc_pointers,
self.q_fixed_size,
self.q_varsize_item_sizes,
@@ -170,9 +162,8 @@
T_IS_GCARRAY_OF_GCPTR = 0x040000
T_IS_WEAKREF = 0x080000
T_IS_RPYTHON_INSTANCE = 0x100000 # the type is a subclass of OBJECT
-T_HAS_FINALIZER = 0x200000
+T_HAS_DESTRUCTOR = 0x200000
T_HAS_CUSTOM_TRACE = 0x400000
-T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
T_KEY_MASK = intmask(0xFF000000)
T_KEY_VALUE = intmask(0x5A000000) # bug detection only
@@ -199,11 +190,9 @@
kind_and_fptr = builder.special_funcptr_for_type(TYPE)
if kind_and_fptr is not None:
kind, fptr = kind_and_fptr
- info.finalizer_or_customtrace = fptr
- if kind == "finalizer":
- infobits |= T_HAS_FINALIZER
- elif kind == 'light_finalizer':
- infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
+ info.destructor_or_customtrace = fptr
+ if kind == "destructor":
+ infobits |= T_HAS_DESTRUCTOR
elif kind == "custom_trace":
infobits |= T_HAS_CUSTOM_TRACE
else:
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -225,7 +225,7 @@
raise RuntimeError(
"a finalizer raised an exception, shouldn't happen")
return llmemory.NULL
- return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+ return llhelper(gctypelayout.GCData.DESTRUCTOR_OR_CT, ll_finalizer)
def make_custom_trace_funcptr_for_type(self, TYPE):
from rpython.memory.gctransform.support import get_rtti
diff --git a/rpython/memory/test/snippet.py b/rpython/memory/test/snippet.py
--- a/rpython/memory/test/snippet.py
+++ b/rpython/memory/test/snippet.py
@@ -115,8 +115,8 @@
return f
- def test_full_finalizer_order(self):
- res = self.run('full_finalizer_order')
+ def test_finalizer_order(self):
+ res = self.run('finalizer_order')
if res != "ok":
i, summary, msg = res.split('\n')
i = int(i)
diff --git a/rpython/memory/test/test_gc.py b/rpython/memory/test/test_gc.py
--- a/rpython/memory/test/test_gc.py
+++ b/rpython/memory/test/test_gc.py
@@ -153,7 +153,7 @@
res = self.interpret(f, [5])
assert res == 6
- def test_finalizer(self):
+ def test_finalizer_simple(self):
class B(object):
pass
b = B()
@@ -303,7 +303,7 @@
res = self.interpret(f, [])
assert res
- def test_weakref_to_object_with_finalizer(self):
+ def test_weakref_to_object_with_destructor(self):
import weakref
class A(object):
count = 0
@@ -323,6 +323,32 @@
res = self.interpret(f, [])
assert res
+ def test_weakref_to_object_with_finalizer(self):
+ import weakref
+ class A(object):
+ count = 0
+ a = A()
+ class B(object):
+ def __init__(self, ref):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
+ # when the finalizer is called, the weakref to myself is
+ # still valid in RPython
+ a.count += 100 + (self.ref() is None)
+ def g():
+ b = B()
+ ref = weakref.ref(b)
+ b.ref = ref
+ return b
+ def f():
+ ref = g()
+ llop.gc__collect(lltype.Void)
+ llop.gc__collect(lltype.Void)
+ result = a.count + 10 * (ref() is None)
+ return result
+ res = self.interpret(f, [])
+ assert res == 110
+
def test_bug_1(self):
import weakref
class B(object):
@@ -348,8 +374,10 @@
count = 0
a = A()
class B(object):
- def __del__(self):
- # when __del__ is called, the weakref to c should be dead
+ def __init__(self, ref):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
+ # when the finalizer is called, the weakref to c should be dead
if self.ref() is None:
a.count += 10 # ok
else:
@@ -371,40 +399,14 @@
res = self.interpret(f, [])
assert res == 11
- def test_weakref_to_object_with_finalizer_ordering(self):
- import weakref
- class A(object):
- count = 0
- a = A()
- class B(object):
- def __del__(self):
- # when __del__ is called, the weakref to myself is still valid
- # in RPython (at least with most GCs; this test might be
- # skipped for specific GCs)
- if self.ref() is self:
- a.count += 10 # ok
- else:
- a.count = 666 # not ok
- def g():
- b = B()
- ref = weakref.ref(b)
- b.ref = ref
- return ref
- def f():
- ref = g()
- llop.gc__collect(lltype.Void)
- llop.gc__collect(lltype.Void)
- result = a.count + (ref() is None)
- return result
- res = self.interpret(f, [])
- assert res == 11
-
def test_weakref_bug_1(self):
import weakref
class A(object):
pass
class B(object):
- def __del__(self):
+ def __init__(self, ref):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
self.wref().x += 1
def g(a):
b = B()
@@ -453,7 +455,8 @@
def __init__(self):
self.id = b.nextid
b.nextid += 1
- def __del__(self):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
b.num_deleted += 1
b.all.append(D(b.num_deleted))
class D(object):
@@ -486,14 +489,16 @@
def __init__(self):
self.id = b.nextid
b.nextid += 1
- def __del__(self):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
+ b.num_deleted += 1
+ self.do_finalizer()
+ def do_finalizer(self):
llop.gc__collect(lltype.Void)
- b.num_deleted += 1
C()
C()
class C(A):
- def __del__(self):
- b.num_deleted += 1
+ def do_finalizer(self):
b.num_deleted_c += 1
def f(x, y):
persistent_a1 = A()
@@ -873,7 +878,8 @@
def __init__(self):
self.id = b.nextid
b.nextid += 1
- def __del__(self):
+ rgc.register_finalizer(self.finalizer)
+ def finalizer(self):
b.num_deleted += 1
def f(x):
a = A()
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -333,7 +333,8 @@
func(obj)
except FinalizeLater:
_finalizer_queue.appendleft((obj, func))
- break
+ return False # interrupted
+ return True # completed
class RegisterFinalizerEntry(ExtRegistryEntry):
_about_ = register_finalizer
@@ -342,8 +343,10 @@
from rpython.annotator import model as annmodel
from rpython.annotator.description import MethodDesc
assert (isinstance(s_method, annmodel.SomePBC) and
- s_method.getKind() is MethodDesc and
- len(s_method.descriptions) == 1)
+ s_method.getKind() is MethodDesc)
+ if len(s_method.descriptions) != 1:
+ raise Exception("rgc.register_finalizer(method): the method must "
+ "not be overridden. Add an indirection if needed")
#
[methoddesc] = s_method.descriptions
key = (register_finalizer, methoddesc.funcdesc, methoddesc.name)
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
