Author: Remi Meier Branch: c7-weakref Changeset: r979:3040d781125a Date: 2014-03-12 16:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/3040d781125a/
Log: start adding weakref support diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -180,6 +180,7 @@ } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); + assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows)); assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_abort)); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -54,6 +54,11 @@ after the object. */ GCFLAG_HAS_SHADOW = 0x04, + /* This flag is set on weakref objects. Weakref objects have a + reference to the referenced object at the byte-offset + stmcb_size_rounded_up(obj) - sizeof(void*) */ + GCFLAG_WEAKREF = 0x08, + /* All remaining bits of the 32-bit 'stm_flags' field are taken by the "overflow number". This is a number that identifies the "overflow objects" from the current transaction among all old @@ -61,7 +66,7 @@ current transaction that have been flushed out of the nursery, which occurs if the same transaction allocates too many objects. */ - GCFLAG_OVERFLOW_NUMBER_bit0 = 0x08 /* must be last */ + GCFLAG_OVERFLOW_NUMBER_bit0 = 0x10 /* must be last */ }; @@ -105,6 +110,14 @@ next minor collection. */ struct tree_s *nursery_objects_shadows; + /* List of all young weakrefs to check in minor collections. These + are the only weakrefs that may point to young objects. */ + struct list_s *young_weakrefs; + + /* List of all old weakrefs to check in major collections. These + weakrefs never point to young objects */ + struct list_s *old_weakrefs; + /* Tree of 'key->callback' associations from stm_call_on_abort() */ struct tree_s *callbacks_on_abort; diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -299,6 +299,9 @@ collect_oldrefs_to_nursery(); + /* now all surviving nursery objects have been moved out */ + stm_move_young_weakrefs(); + throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT)); diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -57,6 +57,8 @@ pr->objects_pointing_to_nursery = NULL; pr->large_overflow_objects = NULL; pr->modified_old_objects = list_create(); + pr->young_weakrefs = list_create(); + pr->old_weakrefs = list_create(); pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); pr->callbacks_on_abort = tree_create(); @@ -95,6 +97,8 @@ assert(pr->objects_pointing_to_nursery == NULL); assert(pr->large_overflow_objects == NULL); list_free(pr->modified_old_objects); + list_free(pr->young_weakrefs); + list_free(pr->old_weakrefs); tree_free(pr->young_outside_nursery); tree_free(pr->nursery_objects_shadows); tree_free(pr->callbacks_on_abort); diff --git a/c7/stm/weakref.c b/c7/stm/weakref.c new file mode 100644 --- /dev/null +++ b/c7/stm/weakref.c @@ -0,0 +1,226 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +object_t *stm_allocate_weakref(ssize_t size_rounded_up) +{ + OPT_ASSERT(size_rounded_up > sizeof(struct object_s)); + object_t *obj = stm_allocate(size_rounded_up); + obj->stm_flags |= GCFLAG_WEAKREF; + LIST_APPEND(STM_PSEGMENT->young_weakrefs, obj); + return obj; +} + + +/***** Minor collection *****/ + +void stm_move_young_weakrefs() +{ + /* The code relies on the fact that no weakref can be an old object + weakly pointing to a young object. Indeed, weakrefs are immutable + so they cannot point to an object that was created after it. + */ + LIST_FOREACH_R( + STM_PSEGMENT->young_weakrefs, + object_t * /*item*/, + ({ + if (_is_in_nursery(item)) { + object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)item; + + /* the following checks are done like in nursery.c: */ + if (!(item->stm_flags & GCFLAG_HAS_SHADOW) + || (pforwarded_array[0] != GCWORD_MOVED)) { + /* weakref dies */ + continue; + } + + item = pforwarded_array[1]; /* moved location */ + } + else { + /* young outside nursery object */ + if (tree_contains(STM_PSEGMENT->young_outside_nursery, + (uintptr_t)item)) { + /* still in the tree -> wasn't seen by the minor collection, + so it doesn't survive */ + continue; + } + } + assert(!_is_young(item)); + + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, item); + ssize_t size = stmcb_size_rounded_up((struct object_s *)realobj); + object_t *pointing_to = *WEAKREF_PTR(item, size); + assert(pointing_to != NULL); + + if (_is_in_nursery(pointing_to)) { + object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)pointing_to; + /* the following checks are done like in nursery.c: */ + if (!(pointing_to->stm_flags & GCFLAG_HAS_SHADOW) + || (pforwarded_array[0] != GCWORD_MOVED)) { + /* pointing_to dies */ + *WEAKREF_PTR(item, size) = NULL; + continue; /* no need to remember in old_weakrefs */ + } + else { + /* moved location */ + *WEAKREF_PTR(item, size) = pforwarded_array[1]; + } + } + else { + /* young outside nursery object or old object */ + if (tree_contains(STM_PSEGMENT->young_outside_nursery, + (uintptr_t)pointing_to)) { + /* still in the tree -> wasn't seen by the minor collection, + so it doesn't survive */ + *WEAKREF_PTR(item, size) = NULL; + continue; /* no need to remember in old_weakrefs */ + } + /* pointing_to was already old */ + } + LIST_APPEND(STM_PSEGMENT->old_weakrefs, item); + })); + list_clear(STM_PSEGMENT->young_weakrefs); +} + + +/***** Major collection *****/ + +/* static _Bool is_partially_visited(gcptr obj) */ +/* { */ +/* /\* Based on gcpage.c:visit_public(). Check the code here if we change */ +/* visit_public(). Returns True or False depending on whether we find any */ +/* version of 'obj' to be MARKED or not. */ +/* *\/ */ +/* assert(IMPLIES(obj->h_tid & GCFLAG_VISITED, */ +/* obj->h_tid & GCFLAG_MARKED)); */ +/* if (obj->h_tid & GCFLAG_MARKED) */ +/* return 1; */ + +/* /\* if (!(obj->h_tid & GCFLAG_PUBLIC)) *\/ */ +/* /\* return 0; *\/ */ +/* assert(!(obj->h_tid & GCFLAG_PREBUILT_ORIGINAL)); */ +/* if (obj->h_original != 0) { */ +/* gcptr original = (gcptr)obj->h_original; */ +/* assert(IMPLIES(original->h_tid & GCFLAG_VISITED, */ +/* original->h_tid & GCFLAG_MARKED)); */ +/* if (original->h_tid & GCFLAG_MARKED) */ +/* return 1; */ +/* } */ +/* return 0; */ +/* } */ + +/* static void update_old_weakrefs_list(struct tx_public_descriptor *gcp) */ +/* { */ +/* long i, size = gcp->old_weakrefs.size; */ +/* gcptr *items = gcp->old_weakrefs.items; */ + +/* for (i = 0; i < size; i++) { */ +/* gcptr weakref = items[i]; */ + +/* /\* if a weakref moved, update its position in the list *\/ */ +/* if (weakref->h_tid & GCFLAG_MOVED) { */ +/* items[i] = (gcptr)weakref->h_original; */ +/* } */ +/* } */ +/* } */ + +/* static void visit_old_weakrefs(struct tx_public_descriptor *gcp) */ +/* { */ +/* /\* Note: it's possible that a weakref points to a public stub to a */ +/* protected object, and only the protected object was marked as */ +/* VISITED so far. In this case, this function needs to mark the */ +/* public stub as VISITED too. */ +/* *\/ */ +/* long i, size = gcp->old_weakrefs.size; */ +/* gcptr *items = gcp->old_weakrefs.items; */ + +/* for (i = 0; i < size; i++) { */ +/* gcptr weakref = items[i]; */ + +/* if (!(weakref->h_tid & GCFLAG_VISITED)) { */ +/* /\* the weakref itself dies *\/ */ +/* } */ +/* else { */ +/* /\* the weakref belongs to our thread, therefore we should */ +/* always see the most current revision here: *\/ */ +/* assert(weakref->h_revision & 1); */ + +/* size_t size = stmgc_size(weakref); */ +/* gcptr pointing_to = *WEAKREF_PTR(weakref, size); */ +/* assert(pointing_to != NULL); */ +/* if (is_partially_visited(pointing_to)) { */ +/* pointing_to = stmgcpage_visit(pointing_to); */ +/* dprintf(("mweakref ptr moved %p->%p\n", */ +/* *WEAKREF_PTR(weakref, size), */ +/* pointing_to)); */ + +/* assert(pointing_to->h_tid & GCFLAG_VISITED); */ +/* *WEAKREF_PTR(weakref, size) = pointing_to; */ +/* } */ +/* else { */ +/* /\* the weakref appears to be pointing to a dying object, */ +/* but we don't know for sure now. Clearing it is left */ +/* to clean_old_weakrefs(). *\/ */ +/* } */ +/* } */ +/* } */ +/* } */ + +/* static void clean_old_weakrefs(struct tx_public_descriptor *gcp) */ +/* { */ +/* long i, size = gcp->old_weakrefs.size; */ +/* gcptr *items = gcp->old_weakrefs.items; */ + +/* for (i = size - 1; i >= 0; i--) { */ +/* gcptr weakref = items[i]; */ +/* assert(weakref->h_revision & 1); */ +/* if (weakref->h_tid & GCFLAG_VISITED) { */ +/* size_t size = stmgc_size(weakref); */ +/* gcptr pointing_to = *WEAKREF_PTR(weakref, size); */ +/* if (pointing_to->h_tid & GCFLAG_VISITED) { */ +/* continue; /\* the target stays alive, the weakref remains *\/ */ +/* } */ +/* dprintf(("mweakref lost ptr %p\n", *WEAKREF_PTR(weakref, size))); */ +/* *WEAKREF_PTR(weakref, size) = NULL; /\* the target dies *\/ */ +/* } */ +/* /\* remove this weakref from the list *\/ */ +/* items[i] = items[--gcp->old_weakrefs.size]; */ +/* } */ +/* gcptrlist_compress(&gcp->old_weakrefs); */ +/* } */ + +/* static void for_each_public_descriptor( */ +/* void visit(struct tx_public_descriptor *)) { */ +/* struct tx_descriptor *d; */ +/* for (d = stm_tx_head; d; d = d->tx_next) */ +/* visit(d->public_descriptor); */ + +/* struct tx_public_descriptor *gcp; */ +/* revision_t index = -1; */ +/* while ((gcp = stm_get_free_public_descriptor(&index)) != NULL) */ +/* visit(gcp); */ +/* } */ + +/* void stm_update_old_weakrefs_lists(void) */ +/* { */ +/* /\* go over old weakrefs lists and update the list with possibly */ +/* new pointers because of copy_over_original *\/ */ +/* for_each_public_descriptor(update_old_weakrefs_list); */ +/* } */ + + +/* void stm_visit_old_weakrefs(void) */ +/* { */ +/* /\* Figure out which weakrefs survive, which possibly */ +/* adds more objects to 'objects_to_trace'. */ +/* *\/ */ +/* for_each_public_descriptor(visit_old_weakrefs); */ +/* } */ + +/* void stm_clean_old_weakrefs(void) */ +/* { */ +/* /\* Clean up the non-surviving weakrefs */ +/* *\/ */ +/* for_each_public_descriptor(clean_old_weakrefs); */ +/* } */ diff --git a/c7/stm/weakref.h b/c7/stm/weakref.h new file mode 100644 --- /dev/null +++ b/c7/stm/weakref.h @@ -0,0 +1,13 @@ +#ifndef _SRCSTM_WEAKREF_H +#define _SRCSTM_WEAKREF_H + + +#define WEAKREF_PTR(wr, sz) ((object_t * TLPREFIX *)(((stm_char *)(wr)) + (sz) - sizeof(void*))) + +void stm_move_young_weakrefs(void); +/* void stm_update_old_weakrefs_lists(void); */ +/* void stm_visit_old_weakrefs(void); */ +/* void stm_clean_old_weakrefs(void); */ + + +#endif diff --git a/c7/stmgc.c b/c7/stmgc.c --- a/c7/stmgc.c +++ b/c7/stmgc.c @@ -12,6 +12,7 @@ #include "stm/contention.h" #include "stm/extra.h" #include "stm/fprintcolor.h" +#include "stm/weakref.h" #include "stm/misc.c" #include "stm/list.c" @@ -28,3 +29,4 @@ #include "stm/contention.c" #include "stm/extra.c" #include "stm/fprintcolor.c" +#include "stm/weakref.c" diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -194,6 +194,14 @@ return (object_t *)p; } +/* Allocate a weakref object. Weakref objects have a + reference to the referenced object at the byte-offset + stmcb_size_rounded_up(obj) - sizeof(void*) + This reference becomes NULL if the referenced object was freed. + You must assign the reference before the next collection may happen. + After that, they may be considered immutable. */ +object_t *stm_allocate_weakref(ssize_t size_rounded_up); + /* stm_setup() needs to be called once at the beginning of the program. stm_teardown() can be called at the end, but that's not necessary diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -25,6 +25,7 @@ void stm_read(object_t *obj); /*void stm_write(object_t *obj); use _checked_stm_write() instead */ object_t *stm_allocate(ssize_t size_rounded_up); +object_t *stm_allocate_weakref(ssize_t size_rounded_up); object_t *_stm_allocate_old(ssize_t size_rounded_up); void stm_setup(void); @@ -54,6 +55,10 @@ void _set_ptr(object_t *obj, int n, object_t *v); object_t * _get_ptr(object_t *obj, int n); +void _set_weakref(object_t *obj, object_t *v); +object_t* _get_weakref(object_t *obj); + + void _stm_start_safe_point(void); bool _check_stop_safe_point(void); @@ -163,6 +168,21 @@ } +#define WEAKREF_PTR(wr, sz) ((object_t * TLPREFIX *)(((stm_char *)(wr)) + (sz) - sizeof(void*))) +void _set_weakref(object_t *obj, object_t *v) +{ + char *realobj = _stm_real_address(obj); + ssize_t size = stmcb_size_rounded_up((struct object_s *)realobj); + *WEAKREF_PTR(obj, size) = v; +} + +object_t * _get_weakref(object_t *obj) +{ + char *realobj = _stm_real_address(obj); + ssize_t size = stmcb_size_rounded_up((struct object_s *)realobj); + return *WEAKREF_PTR(obj, size); +} + void _set_ptr(object_t *obj, int n, object_t *v) { long nrefs = (long)((myobj_t*)obj)->type_id - 421420; @@ -266,6 +286,16 @@ lib._set_type_id(o, tid) return o +def stm_allocate_weakref(point_to_obj): + o = lib.stm_allocate(HDR + WORD) + tid = 421420 + lib._set_type_id(o, tid) + lib._set_weakref(o, point_to_obj) + return o + +def stm_get_weakref(o): + return lib._get_weakref(o) + def stm_allocate_refs(n): o = lib.stm_allocate(HDR + n * WORD) tid = 421420 + n diff --git a/c7/test/test_weakref.py b/c7/test/test_weakref.py new file mode 100644 --- /dev/null +++ b/c7/test/test_weakref.py @@ -0,0 +1,221 @@ +import py +from support import * + + + + +class TestMinorCollection(BaseTest): + def test_simple(self): + lib._stm_set_nursery_free_count(2048) + self.start_transaction() + + self.push_root_no_gc() + lp1 = stm_allocate_weakref(ffi.NULL) # no collection here + self.pop_root() + + assert stm_get_weakref(lp1) == ffi.NULL + + self.push_root(lp1) + stm_minor_collect() + lp1 = self.pop_root() + + assert stm_get_weakref(lp1) == ffi.NULL + + # def test_weakref_invalidate(self): + # p2 = nalloc(HDR) + # p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) + # assert p1.h_tid == WEAKREF_TID | GCFLAG_IMMUTABLE | GCFLAG_WEAKREF + # assert p1.h_revision == lib.get_private_rev_num() + # assert lib.rawgetptr(p1, 0) == p2 + # lib.stm_push_root(p1) + # minor_collect() + # p1 = lib.stm_pop_root() + # assert lib.rawgetptr(p1, 0) == ffi.NULL + + # def test_weakref_itself_dies(self): + # p2 = nalloc(HDR) + # p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) + # minor_collect() + + # def test_weakref_keep(self): + # p2 = nalloc(HDR) + # p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) + # assert p1.h_tid == WEAKREF_TID | GCFLAG_IMMUTABLE | GCFLAG_WEAKREF + # assert p1.h_revision == lib.get_private_rev_num() + # assert lib.rawgetptr(p1, 0) == p2 + # lib.stm_push_root(p1) + # lib.stm_push_root(p2) + # minor_collect() + # p2 = lib.stm_pop_root() + # p1 = lib.stm_pop_root() + # assert lib.rawgetptr(p1, 0) == p2 + + # def test_weakref_old_keep(self): + # p2 = oalloc(HDR) + # p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) + # assert p1.h_tid == WEAKREF_TID | GCFLAG_IMMUTABLE | GCFLAG_WEAKREF + # assert p1.h_revision == lib.get_private_rev_num() + # assert lib.rawgetptr(p1, 0) == p2 + # lib.stm_push_root(p1) + # lib.stm_push_root(p2) + # minor_collect() + # p2 = lib.stm_pop_root() + # p1 = lib.stm_pop_root() + # assert lib.rawgetptr(p1, 0) == p2 + + + # def test_old_private_not_keep_alive_weakref(self): + # p = palloc(HDR + WORD) + # q = palloc_refs(1) + + # def f1(c): + # if c == 1: + # # currently fails because: + # # p1 still in old_objects_to_trace + # # -> keeps alive weakp1w + # # -> stm_move_young_weakrefs() sees a weakref pointing + # # to an aborted object + # minor_collect() + # return + + # # allocate the "container" as old, private q1 + # q1 = lib.stm_write_barrier(q) + # assert classify(q1) == "private" + # lib.stm_push_root(q1) + # minor_collect() + # q1 = lib.stm_pop_root() + # assert classify(q1) == "private" + # assert q1.h_tid & GCFLAG_OLD + # assert q1.h_tid & GCFLAG_WRITE_BARRIER + + # # allocate young private p1 to point to + # p1 = lib.stm_write_barrier(p) + # assert ffi.cast("gcptr", p1.h_original) == p + # assert classify(p1) == "private" + # assert not (p1.h_tid & GCFLAG_OLD) + + # lib.stm_push_root(p1) + # lib.stm_push_root(q1) + # weakp1w = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p1) + # q1 = lib.stm_pop_root() + # p1 = lib.stm_pop_root() + # # q1 still old, p1 still young, weakp1w also young + + # q1w = lib.stm_write_barrier(q1) + # # add q1 to old_objects_to_trace + # assert q1 == q1w # was and is private + # lib.rawsetptr(q1, 0, weakp1w) + + # abort_and_retry() + + # perform_transaction(f1) + + + + + + + + + + +# class TestMajorCollection(BaseTest): + +# def test_weakref_old(self): +# p2 = nalloc(HDR) +# p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) +# # +# lib.stm_push_root(p1) +# lib.stm_push_root(p2) +# major_collect() +# p2 = lib.stm_pop_root() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == p2 +# # +# lib.stm_push_root(p1) +# major_collect() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == ffi.NULL + +# def test_weakref_to_prebuilt(self): +# p2 = palloc(HDR) +# p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) +# # +# lib.stm_push_root(p1) +# major_collect() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == p2 + +# def test_weakref_update_version(self): +# p2 = oalloc(HDR + WORD); make_public(p2) +# p1 = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, p2) +# # +# lib.stm_push_root(p1) +# lib.stm_push_root(p2) +# major_collect() +# p2 = lib.stm_pop_root() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == p2 +# # +# lib.stm_commit_transaction() +# lib.stm_begin_inevitable_transaction() +# # +# lib.setlong(p2, 0, 912809218) # write barrier +# assert lib.rawgetlong(p2, 0) == 0 +# lib.stm_push_root(p1) +# lib.stm_push_root(p2) +# major_collect() +# p2 = lib.stm_pop_root() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == p2 +# assert lib.rawgetlong(p2, 0) == 0 +# # +# lib.stm_commit_transaction() +# lib.stm_begin_inevitable_transaction() +# # +# assert lib.rawgetptr(p1, 0) == p2 +# assert lib.rawgetlong(p2, 0) == 0 +# lib.stm_push_root(p1) +# lib.stm_push_root(p2) +# major_collect() +# p2b = lib.stm_pop_root() +# p1 = lib.stm_pop_root() +# assert lib.rawgetptr(p1, 0) == p2 +# assert p2b != p2 +# assert lib.getlong(p2b, 0) == 912809218 +# assert lib.getlong(p2, 0) == 912809218 + + +# def test_stealing(self): +# p = palloc_refs(1) +# u = palloc_refs(1) + +# def f1(r): +# q = nalloc(HDR+WORD) +# # lib.stm_push_root(q) +# w = lib.stm_weakref_allocate(WEAKREF_SIZE, WEAKREF_TID, q) +# # q = lib.stm_pop_root() +# setptr(p, 0, w) +# setptr(u, 0, q) +# minor_collect() +# lib.stm_commit_transaction() +# lib.stm_begin_inevitable_transaction() +# r.set(2) +# r.wait(3) +# print "happy" + +# def f2(r): +# r.wait(2) +# # steal p, should stub the weakref contained in it +# pr = lib.stm_read_barrier(p) +# w = rawgetptr(pr, 0) +# assert classify(w) == "stub" + +# # read weakref, should stub out weakptr +# wr = lib.stm_read_barrier(w) +# assert wr.h_tid & GCFLAG_WEAKREF +# assert classify(lib.rawgetptr(wr, 0)) == "stub" + +# r.set(3) + +# run_parallel(f1, f2) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit