Author: Remi Meier <[email protected]>
Branch: c8-private-pages
Changeset: r1561:72a83d94df8e
Date: 2015-01-21 13:03 +0100
http://bitbucket.org/pypy/stmgc/changeset/72a83d94df8e/
Log: Implement what overflow objs in c7 do using WB_EXECUTED. This undoes
part of the previous commit which should not be necessary anymore.
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -705,6 +705,7 @@
}
assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
+ assert(list_is_empty(STM_PSEGMENT->new_objects));
assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
@@ -756,6 +757,7 @@
STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
STM_PSEGMENT->transaction_state = TS_NONE;
list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+ list_clear(STM_PSEGMENT->new_objects);
release_thread_segment(tl);
/* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -775,6 +777,20 @@
#endif
}
+static void push_new_objects_to_other_segments(void)
+{
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
+ ({
+ assert(item->stm_flags & GCFLAG_WB_EXECUTED);
+ item->stm_flags &= ~GCFLAG_WB_EXECUTED;
+ synchronize_object_enqueue(item);
+ }));
+ synchronize_objects_flush();
+ release_privatization_lock(STM_SEGMENT->segment_num);
+}
+
+
void stm_commit_transaction(void)
{
assert(!_has_mutex());
@@ -784,6 +800,8 @@
dprintf(("> stm_commit_transaction()\n"));
minor_collection(1);
+ push_new_objects_to_other_segments();
+
_validate_and_add_to_commit_log();
invoke_and_clear_user_callbacks(0); /* for commit */
@@ -887,6 +905,7 @@
tl->last_abort__bytes_in_nursery = bytes_in_nursery;
list_clear(pseg->objects_pointing_to_nursery);
+ list_clear(pseg->new_objects);
#pragma pop_macro("STM_SEGMENT")
#pragma pop_macro("STM_PSEGMENT")
}
@@ -999,6 +1018,8 @@
assert(!_is_young(obj));
assert(STM_PSEGMENT->privatization_lock);
assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+ assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+
ssize_t obj_size = stmcb_size_rounded_up(
(struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
OPT_ASSERT(obj_size >= 16);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -74,11 +74,20 @@
struct tree_s *young_outside_nursery;
struct tree_s *nursery_objects_shadows;
+ /* list of objects created in the current transaction and
+ that survived at least one minor collection. They need
+ to be synchronized to other segments on commit, but they
+ do not need to be in the commit log entry. */
+ struct list_s *new_objects;
+
uint8_t privatization_lock; // XXX KILL
uint8_t safe_point;
uint8_t transaction_state;
+ /* Temp for minor collection */
+ bool minor_collect_will_commit_now;
+
struct tree_s *callbacks_on_commit_and_abort[2];
struct stm_commit_log_entry_s *last_commit_log_entry;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -192,6 +192,14 @@
/************************************************************/
+
+static bool is_new_object(object_t *obj)
+{
+ struct object_s *realobj = (struct
object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */
+ return realobj->stm_flags & GCFLAG_WB_EXECUTED;
+}
+
+
static inline void mark_record_trace(object_t **pobj)
{
/* takes a normal pointer to a thread-local pointer to an object */
@@ -226,11 +234,12 @@
stmcb_trace(realobj, &mark_record_trace);
/* trace all references found in sharing seg0 (should always be
- up-to-date and not cause segfaults) */
+ up-to-date and not cause segfaults, except for new objs) */
while (!list_is_empty(marked_objects_to_trace)) {
obj = (object_t *)list_pop_item(marked_objects_to_trace);
- realobj = (struct object_s *)REAL_ADDRESS(stm_object_pages, obj);
+ char *base = is_new_object(obj) ? segment_base : stm_object_pages;
+ realobj = (struct object_s *)REAL_ADDRESS(base, obj);
stmcb_trace(realobj, &mark_record_trace);
}
}
@@ -243,14 +252,31 @@
mark_and_trace(obj, segment_base);
}
+
+static void mark_visit_possibly_new_object(char *segment_base, object_t *obj)
+{
+ /* if newly allocated object, we trace in segment_base, otherwise in
+ the sharing seg0 */
+ if (obj == NULL)
+ return;
+
+ if (is_new_object(obj)) {
+ mark_visit_object(obj, segment_base);
+ } else {
+ mark_visit_object(obj, stm_object_pages);
+ }
+}
+
static void *mark_visit_objects_from_ss(void *_, const void *slice, size_t
size)
{
const struct stm_shadowentry_s *p, *end;
p = (const struct stm_shadowentry_s *)slice;
end = (const struct stm_shadowentry_s *)(slice + size);
for (; p < end; p++)
- if ((((uintptr_t)p->ss) & 3) == 0)
+ if ((((uintptr_t)p->ss) & 3) == 0) {
+ assert(!is_new_object(p->ss));
mark_visit_object(p->ss, stm_object_pages); // seg0
+ }
return NULL;
}
@@ -273,6 +299,7 @@
}
+
static void mark_visit_from_modified_objects(void)
{
/* look for modified objects in segments and mark all of them
@@ -280,53 +307,29 @@
some of the pages) */
long i;
- struct list_s *uniques = list_create();
for (i = 1; i < NB_SEGMENTS; i++) {
char *base = get_segment_base(i);
- OPT_ASSERT(list_is_empty(uniques));
- /* the list of modified_old_objs can be huge and contain a lot
- of duplicated (same obj, different slice) entries. It seems
- worth it to build a new list without duplicates.
- The reason is that newly created objs, when moved out of the
- nursery, don't have WB_EXECUTED flag. Thus we execute waaay
- too many write barriers per transaction and add them all
- to this list (and the commit log). XXXXX */
struct list_s *lst = get_priv_segment(i)->modified_old_objects;
+ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(lst->items +
lst->count);
- struct stm_undo_s *undo = (struct stm_undo_s *)lst->items;
- struct stm_undo_s *end = (struct stm_undo_s *)(lst->items +
lst->count);
- for (; undo < end; undo++) {
- object_t *obj = undo->object;
- struct object_s *dst = (struct object_s*)REAL_ADDRESS(base, obj);
+ for (; modified < end; modified++) {
+ object_t *obj = modified->object;
+ /* All modified objs have all pages accessible for now.
+ This is because we create a backup of the whole obj
+ and thus make all pages accessible. */
+ assert_obj_accessible_in(i, obj);
- if (!(dst->stm_flags & GCFLAG_VISITED)) {
- LIST_APPEND(uniques, obj);
- dst->stm_flags |= GCFLAG_VISITED;
+ assert(!is_new_object(obj)); /* should never be in that list */
+
+ if (!mark_visited_test_and_set(obj)) {
+ /* trace shared, committed version */
+ mark_and_trace(obj, stm_object_pages);
}
+ mark_and_trace(obj, base); /* private, modified version */
}
-
-
- LIST_FOREACH_R(uniques, object_t*,
- ({
- /* clear the VISITED flags again and actually visit them */
- struct object_s *dst = (struct object_s*)REAL_ADDRESS(base,
item);
- dst->stm_flags &= ~GCFLAG_VISITED;
-
- /* All modified objs have all pages accessible for now.
- This is because we create a backup of the whole obj
- and thus make all pages accessible. */
- assert_obj_accessible_in(i, item);
-
- if (!mark_visited_test_and_set(item))
- mark_and_trace(item, stm_object_pages); /* shared,
committed version */
- mark_and_trace(item, base); /* private, modified
version */
- }));
-
-
- list_clear(uniques);
}
- LIST_FREE(uniques);
}
static void mark_visit_from_roots(void)
@@ -341,19 +344,29 @@
/* look at all objs on the shadow stack (they are old but may
be uncommitted so far, so only exist in the associated_segment_num).
- However, since we just executed a minor collection, they were
+ IF they are uncommitted new objs, trace in the actual segment,
+ otherwise, since we just executed a minor collection, they were
all synced to the sharing seg0. Thus we can trace them there.
If they were again modified since then, they were traced
by mark_visit_from_modified_object() already.
*/
+
+ /* only for new, uncommitted objects:
+ If 'tl' is currently running, its 'associated_segment_num'
+ field is the segment number that contains the correct
+ version of its overflowed objects. */
+ char *segment_base = get_segment_base(tl->associated_segment_num);
+
struct stm_shadowentry_s *current = tl->shadowstack;
struct stm_shadowentry_s *base = tl->shadowstack_base;
while (current-- != base) {
- if ((((uintptr_t)current->ss) & 3) == 0)
- mark_visit_object(current->ss, stm_object_pages);
+ if ((((uintptr_t)current->ss) & 3) == 0) {
+ mark_visit_possibly_new_object(segment_base, current->ss);
+ }
}
- mark_visit_object(tl->thread_local_obj, stm_object_pages);
+
+ mark_visit_possibly_new_object(segment_base, tl->thread_local_obj);
tl = tl->next;
} while (tl != stm_all_thread_locals);
@@ -362,9 +375,10 @@
long i;
for (i = 1; i < NB_SEGMENTS; i++) {
if (get_priv_segment(i)->transaction_state != TS_NONE) {
- mark_visit_object(
- get_priv_segment(i)->threadlocal_at_start_of_transaction,
- stm_object_pages);
+ mark_visit_possibly_new_object(
+ get_segment_base(i),
+ get_priv_segment(i)->threadlocal_at_start_of_transaction);
+
stm_rewind_jmp_enum_shadowstack(
get_segment(i)->running_thread,
mark_visit_objects_from_ss);
@@ -372,6 +386,46 @@
}
}
+static void ready_new_objects(void)
+{
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+ /* objs in new_objects only have garbage in the sharing seg0,
+ since it is used to mark objs as visited, we must make
+ sure the flag is cleared at the start of a major collection.
+ (XXX: ^^^ may be optional if we have the part below)
+
+ Also, we need to be able to recognize these objects in order
+ to only trace them in the segment they are valid in. So we
+ also make sure to set WB_EXECUTED in the sharing seg0. No
+ other objs than new_objects have WB_EXECUTED in seg0 (since
+ there can only be committed versions there).
+ */
+
+ long i;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+ struct list_s *lst = pseg->new_objects;
+
+ LIST_FOREACH_R(lst, object_t* /*item*/,
+ ({
+ struct object_s *realobj;
+ /* WB_EXECUTED always set in this segment */
+ assert(realobj = (struct
object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
+ assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
+
+ /* clear VISITED and ensure WB_EXECUTED in seg0 */
+ mark_visited_test_and_clear(item);
+ realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages,
item);
+ realobj->stm_flags |= GCFLAG_WB_EXECUTED;
+ }));
+ }
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
+}
+
static void clean_up_segment_lists(void)
{
@@ -411,6 +465,16 @@
we "didn't do a collection" at all. So nothing to do on
modified_old_objs. */
}
+
+ /* remove from new_objects all objects that die */
+ lst = pseg->new_objects;
+ uintptr_t n = list_count(lst);
+ while (n-- > 0) {
+ object_t *obj = (object_t *)list_item(lst, n);
+ if (!mark_visited_test(obj)) {
+ list_set_item(lst, n, list_pop_item(lst));
+ }
+ }
}
#pragma pop_macro("STM_SEGMENT")
#pragma pop_macro("STM_PSEGMENT")
@@ -486,6 +550,8 @@
DEBUG_EXPECT_SEGFAULT(false);
+ ready_new_objects();
+
/* marking */
LIST_CREATE(marked_objects_to_trace);
mark_visit_from_modified_objects();
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -125,8 +125,15 @@
/* a young object outside the nursery */
nobj = obj;
tree_delete_item(STM_PSEGMENT->young_outside_nursery, (uintptr_t)nobj);
+ nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
+ }
- nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
+ /* if this is not during commit, we will add them to the new_objects
+ list and push them to other segments on commit. Thus we can add
+ the WB_EXECUTED flag so that they don't end up in modified_old_objects
*/
+ assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED));
+ if (!STM_PSEGMENT->minor_collect_will_commit_now) {
+ nobj->stm_flags |= GCFLAG_WB_EXECUTED;
}
/* Must trace the object later */
@@ -192,20 +199,31 @@
_collect_now(obj);
if (obj_sync_now & FLAG_SYNC_LARGE) {
- /* this is a newly allocated object. We must synchronize it
- to other segments (after we added WRITE_BARRIER). */
- acquire_privatization_lock(STM_SEGMENT->segment_num);
- synchronize_object_enqueue(obj);
- release_privatization_lock(STM_SEGMENT->segment_num);
+ /* this is a newly allocated obj in this transaction. We must
+ either synchronize the object to other segments now, or
+ add the object to new_objects list */
+ if (STM_PSEGMENT->minor_collect_will_commit_now) {
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ synchronize_object_enqueue(obj);
+ release_privatization_lock(STM_SEGMENT->segment_num);
+ } else {
+ LIST_APPEND(STM_PSEGMENT->new_objects, obj);
+ }
}
/* the list could have moved while appending */
lst = STM_PSEGMENT->objects_pointing_to_nursery;
}
- acquire_privatization_lock(STM_SEGMENT->segment_num);
- synchronize_objects_flush();
- release_privatization_lock(STM_SEGMENT->segment_num);
+ /* flush all new objects to other segments now */
+ if (STM_PSEGMENT->minor_collect_will_commit_now) {
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ synchronize_objects_flush();
+ release_privatization_lock(STM_SEGMENT->segment_num);
+ } else {
+ /* nothing in the queue when not committing */
+ assert(STM_PSEGMENT->sq_len == 0);
+ }
}
@@ -273,6 +291,8 @@
{
dprintf(("minor_collection commit=%d\n", (int)commit));
+ STM_PSEGMENT->minor_collect_will_commit_now = commit;
+
collect_roots_in_nursery();
collect_oldrefs_to_nursery();
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -95,6 +95,7 @@
pr->pub.segment_num = i;
pr->pub.segment_base = segment_base;
pr->modified_old_objects = list_create();
+ pr->new_objects = list_create();
pr->objects_pointing_to_nursery = list_create();
pr->young_outside_nursery = tree_create();
pr->nursery_objects_shadows = tree_create();
@@ -134,6 +135,8 @@
assert(list_is_empty(pr->objects_pointing_to_nursery));
list_free(pr->objects_pointing_to_nursery);
list_free(pr->modified_old_objects);
+ assert(list_is_empty(pr->new_objects));
+ list_free(pr->new_objects);
tree_free(pr->young_outside_nursery);
tree_free(pr->nursery_objects_shadows);
tree_free(pr->callbacks_on_commit_and_abort[0]);
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
