Author: Armin Rigo <[email protected]>
Branch: c7-more-segments
Changeset: r1051:234b6a0ba9a5
Date: 2014-03-16 17:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/234b6a0ba9a5/
Log: Resharing pages, still buggy, in-progress
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -263,7 +263,62 @@
return false;
}
-static void synchronize_object_now(object_t *obj, bool assume_local_private)
+static void copy_object_to_shared(object_t *obj, int source_segment_num)
+{
+ /* Only used by major GC. XXX There is a lot of code duplication
+ with synchronize_object_now() but I don't completely see how to
+ improve...
+ */
+ assert(_has_mutex_pages());
+ assert(!_is_young(obj));
+
+ uintptr_t start = (uintptr_t)obj;
+ uintptr_t first_page = start / 4096UL;
+ struct object_s *realobj = (struct object_s *)
+ REAL_ADDRESS(get_segment(source_segment_num)->segment_base, obj);
+
+ if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) {
+ abort();//XXX WRITE THE FAST CASE
+ }
+ else {
+ ssize_t obj_size = stmcb_size_rounded_up(realobj);
+ assert(obj_size >= 16);
+ uintptr_t end = start + obj_size;
+ uintptr_t last_page = (end - 1) / 4096UL;
+
+ for (; first_page <= last_page; first_page++) {
+
+ /* Copy the object into the shared page, if needed */
+ if (is_private_page(source_segment_num, first_page)) {
+
+ uintptr_t copy_size;
+ if (first_page == last_page) {
+ /* this is the final fragment */
+ copy_size = end - start;
+ }
+ else {
+ /* this is a non-final fragment, going up to the
+ page's end */
+ copy_size = 4096 - (start & 4095);
+ }
+ /* double-check that the result fits in one page */
+ assert(copy_size > 0);
+ assert(copy_size + (start & 4095) <= 4096);
+
+ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start);
+ char *dst = REAL_ADDRESS(stm_object_pages, start);
+ if (copy_size == 4096)
+ pagecopy(dst, src);
+ else
+ memcpy(dst, src, copy_size);
+ }
+
+ start = (start + 4096) & ~4095;
+ }
+ }
+}
+
+static void synchronize_object_now(object_t *obj)
{
/* Copy around the version of 'obj' that lives in our own segment.
It is first copied into the shared pages, and then into other
@@ -306,12 +361,9 @@
assert(copy_size + (start & 4095) <= 4096);
/* First copy the object into the shared page, if needed */
- assert(IMPLY(assume_local_private,
- is_private_page(myself, first_page)));
-
char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start);
char *dst = REAL_ADDRESS(stm_object_pages, start);
- if (assume_local_private || is_private_page(myself, first_page)) {
+ if (is_private_page(myself, first_page)) {
if (copy_size == 4096)
pagecopy(dst, src);
else
@@ -352,7 +404,7 @@
return;
LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
- synchronize_object_now(item, false));
+ synchronize_object_now(item));
}
static void push_modified_to_other_segments(void)
@@ -374,7 +426,7 @@
/* copy the object to the shared page, and to the other
private pages as needed */
- synchronize_object_now(item, true);
+ synchronize_object_now(item);
}));
list_clear(STM_PSEGMENT->modified_old_objects);
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -224,4 +224,5 @@
asm("/* workaround for llvm bug */");
}
-static void synchronize_object_now(object_t *obj, bool assume_local_private);
+static void copy_object_to_shared(object_t *obj, int source_segment_num);
+static void synchronize_object_now(object_t *obj);
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -192,130 +192,129 @@
/************************************************************/
+static uintptr_t object_last_page(object_t *obj)
+{
+ uintptr_t lastbyte;
+ struct object_s *realobj =
+ (struct object_s *)REAL_ADDRESS(stm_object_pages, obj);
-#if 0
-static inline void mark_single_flag_private(uintptr_t pagenum)
-{
- if (flag_page_private[pagenum] == PRIVATE_PAGE) {
- assert(pagenum >= END_NURSERY_PAGE);
- assert(pagenum < NB_PAGES);
- flag_page_private[pagenum] = SEGMENT1_PAGE;
+ if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) {
+ lastbyte = (uintptr_t)obj;
}
else {
- assert(flag_page_private[pagenum] == SHARED_PAGE ||
- flag_page_private[pagenum] == SEGMENT1_PAGE);
+ /* get the size of the object */
+ size_t obj_size = stmcb_size_rounded_up(realobj);
+
+ /* that's the last byte within the object */
+ lastbyte = ((uintptr_t)obj) + obj_size - 1;
}
+ return lastbyte / 4096UL;
}
-static inline void mark_flag_page_private(object_t *obj, char *segment_base)
+/* A macro that expands to: run the 'expression' for every page that
+ touches objects in the 'modified_old_objects' list.
+*/
+#define BITOP(expression) \
+ LIST_FOREACH_R( \
+ get_priv_segment(segment_num)->modified_old_objects, \
+ object_t * /* item */, \
+ ({ \
+ struct page_shared_s *ps; \
+ uintptr_t pagenum = ((uintptr_t)item) / 4096UL; \
+ uintptr_t count = object_last_page(item) - pagenum; \
+ ps = &pages_privatized[pagenum - PAGE_FLAG_START]; \
+ do { \
+ expression; \
+ ps++; \
+ } while (count--); \
+ }));
+
+static void major_hide_private_bits_for_modified_objects(long segment_num)
{
- uintptr_t first_page = ((uintptr_t)obj) / 4096UL;
-
- if (LIKELY((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0)) {
- mark_single_flag_private(first_page);
- }
- else {
- char *realobj;
- size_t obj_size;
- uintptr_t end_page;
-
- /* get the size of the object */
- realobj = REAL_ADDRESS(segment_base, obj);
- obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
-
- /* that's the page *following* the last page with the object */
- end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL;
-
- while (first_page < end_page)
- mark_single_flag_private(first_page++);
- }
+ uint64_t negativebitmask = ~(1 << (segment_num - 1));
+#ifndef NDEBUG
+ BITOP(assert((ps->by_segment & negativebitmask) != ps->by_segment));
+#endif
+ BITOP(ps->by_segment &= negativebitmask);
}
-static void major_reshare_pages_range(uintptr_t first_page, uintptr_t end_page)
+static void major_restore_private_bits_for_modified_objects(long segment_num)
{
- uintptr_t i;
- for (i = first_page; i < end_page; i++) {
+ uint64_t positivebitmask = 1 << (segment_num - 1);
+ BITOP(ps->by_segment |= positivebitmask);
+}
- switch (flag_page_private[i]) {
-
- case SEGMENT1_PAGE:
- /* this page stays private after major collection */
- flag_page_private[i] = PRIVATE_PAGE;
- break;
-
- case PRIVATE_PAGE:;
- /* this page becomes shared again. No object in it was
- traced belonging to a segment other than 0.
-
- XXX This is maybe a too-strict condition, but the more
- general condition "all traced objects belong to the same
- segment" has problems with large objects in segments > 0.
- More precisely: we'd need to keep in the shared page the
- content of the objects (from segment > 0), but also the
- largemalloc's chunk data (stored in segment 0).
- */
-#if NB_SEGMENTS != 2
-# error "limited to NB_SEGMENTS == 2"
-#endif
- char *ppage0 = get_segment_base(0) + i * 4096;
- char *ppage1 = get_segment_base(1) + i * 4096;
-
- /* two cases for mapping pages to file-pages (fpages):
- - (0->0, 1->1)
- - (0->1, 1->0)
- Distinguish which case it is by hacking a lot */
-
- // 0->0,1->1 or 0->1,1->0
- /* map page 1 to fpage 0: */
- d_remap_file_pages(ppage1, 4096, i);
- // 0->0,1->0 or 0->1,1->0
-
- char oldvalue0 = *ppage0;
- char oldvalue1 = *ppage1;
- asm("":::"memory");
- *ppage0 = 1 + oldvalue1;
- asm("":::"memory");
- char newvalue1 = *ppage1;
- asm("":::"memory");
- *ppage0 = oldvalue0;
- /* if we are in 0->0,1->0, old and new are different:
- In this case we are done. We keep the largemalloc
- data structure and objects of ppage0/fpage0 */
- if (oldvalue1 == newvalue1) {
- // 0->1,1->0
- /* ppage0/fpage1 has the data structure that we want
- in ppage1/fpage0, so we copy it */
- pagecopy(ppage1, ppage0); // copy from page0 to page1,
- // i.e. from the underlying memory seg1 to seg0
- d_remap_file_pages(ppage0, 4096, i);
- // 0->0,1->0
- }
- flag_page_private[i] = SHARED_PAGE;
-
- increment_total_allocated(-4096 * (NB_SEGMENTS-1));
- break;
-
- case SHARED_PAGE:
- break; /* stay shared */
-
- default:
- assert(!"unexpected flag_page_private");
- }
- }
-}
+#undef BITOP
static void major_reshare_pages(void)
{
/* re-share pages if possible. Each re-sharing decreases
total_allocated by 4096. */
- major_reshare_pages_range(
- END_NURSERY_PAGE, /* not the nursery! */
- (uninitialized_page_start - stm_object_pages) / 4096UL);
- major_reshare_pages_range(
- (uninitialized_page_stop - stm_object_pages) / 4096UL,
- NB_PAGES);
+
+ long i;
+ mutex_pages_lock();
+ for (i = 1; i <= NB_SEGMENTS; i++) {
+
+ /* For each segment, push the current overflow objects from
+ private pages to the corresponding shared pages, if necessary.
+ */
+ struct list_s *lst = get_priv_segment(i)->large_overflow_objects;
+ if (lst != NULL) {
+ LIST_FOREACH_R(lst, object_t *, copy_object_to_shared(item, i));
+ }
+
+ /* The 'modified_old_objects' list gives the list of objects
+ whose pages need to remain private. We temporarily remove
+ these bits from 'pages_privatized', so that these pages will
+ be skipped by the loop below.
+ */
+ major_hide_private_bits_for_modified_objects(i);
+ }
+
+ /* Now loop over all pages that are still in 'pages_privatized',
+ and re-share them.
+ */
+ uintptr_t pagenum, endpagenum;
+ struct page_shared_s ps;
+ pagenum = END_NURSERY_PAGE; /* starts after the nursery */
+ endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL;
+
+ while (1) {
+ if (UNLIKELY(pagenum == endpagenum)) {
+ /* we reach this point usually twice, because there are
+ more pages after 'uninitialized_page_stop' */
+ if (endpagenum == NB_PAGES)
+ break; /* done */
+ pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL;
+ endpagenum = NB_PAGES;
+ if (pagenum == endpagenum)
+ break; /* no pages in the 2nd section, so done too */
+ }
+
+ ps = pages_privatized[pagenum - PAGE_FLAG_START];
+ if (ps.by_segment != 0) {
+ pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0;
+
+ long j;
+ for (j = 0; j < NB_SEGMENTS; j++) {
+ if (ps.by_segment & (1 << j)) {
+ /* Page 'pagenum' is private in segment 'j + 1'. Reshare */
+ page_reshare(j + 1, pagenum);
+ }
+ }
+ }
+ pagenum++;
+ }
+
+ /* Done. Now 'pages_privatized' should be entirely zeroes. Restore
+ the previously-hidden bits
+ */
+ for (i = 1; i <= NB_SEGMENTS; i++) {
+ major_restore_private_bits_for_modified_objects(i);
+ }
+ mutex_pages_unlock();
}
-#endif
+
/************************************************************/
@@ -472,7 +471,9 @@
static void sweep_large_objects(void)
{
+ mutex_pages_lock();
_stm_largemalloc_sweep();
+ mutex_pages_unlock();
}
static void clean_write_locks(void)
@@ -488,7 +489,7 @@
memset(write_locks + lock2_idx, 0, sizeof(write_locks) - lock2_idx);
}
-static void major_set_write_locks(void)
+static void major_restore_write_locks(void)
{
/* restore the write locks on the modified objects */
long i;
@@ -518,6 +519,10 @@
dprintf((" | used before collection: %ld\n",
(long)pages_ctl.total_allocated));
+ /* reshare pages */
+ if (RESHARE_PAGES)
+ major_reshare_pages();
+
/* marking */
LIST_CREATE(mark_objects_to_trace);
mark_visit_from_modified_objects();
@@ -531,17 +536,11 @@
clean_up_segment_lists();
/* sweeping */
- mutex_pages_lock();
-#if 0
- if (RESHARE_PAGES)
- major_reshare_pages();
-#endif
sweep_large_objects();
//sweep_uniform_pages();
- mutex_pages_unlock();
clean_write_locks();
- major_set_write_locks();
+ major_restore_write_locks();
dprintf((" | used after collection: %ld\n",
(long)pages_ctl.total_allocated));
diff --git a/c7/stm/gcpage.h b/c7/stm/gcpage.h
--- a/c7/stm/gcpage.h
+++ b/c7/stm/gcpage.h
@@ -15,7 +15,7 @@
#define GC_MAJOR_COLLECT 1.82
/* re-share pages after major collections (1 or 0) */
-#define RESHARE_PAGES 0
+#define RESHARE_PAGES 1
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -200,7 +200,7 @@
*/
if (STM_PSEGMENT->minor_collect_will_commit_now) {
mutex_pages_lock();
- synchronize_object_now(obj, false);
+ synchronize_object_now(obj);
mutex_pages_unlock();
}
else
diff --git a/c7/stm/pages.c b/c7/stm/pages.c
--- a/c7/stm/pages.c
+++ b/c7/stm/pages.c
@@ -140,6 +140,33 @@
mutex_pages_unlock();
}
+static void page_reshare(long segment_num, uintptr_t pagenum)
+{
+ char *segment_base = get_segment(segment_num)->segment_base;
+
+#if 0 /* disabled: the private page that we are removing is
+ typically missing the inter-object information from
+ largemalloc.c */
+ long i, errors=0;
+ uint64_t *p = (uint64_t *)(stm_object_pages + pagenum * 4096UL);
+ uint64_t *q = (uint64_t *)(segment_base + pagenum * 4096UL);
+ for (i = 0; i < 4096 / 8; i++) {
+ if (p[i] != q[i]) {
+ fprintf(stderr, "%p: 0x%lx\t\t%p: 0x%lx\n",
+ &p[i], p[i], &q[i], q[i]);
+ errors++;
+ }
+ }
+ assert(!errors);
+#endif
+
+ madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED);
+ d_remap_file_pages(segment_base + pagenum * 4096UL,
+ 4096, pagenum);
+ increment_total_allocated(-4096);
+}
+
+
#if 0
static bool is_fully_in_shared_pages(object_t *obj)
{
diff --git a/c7/stm/pages.h b/c7/stm/pages.h
--- a/c7/stm/pages.h
+++ b/c7/stm/pages.h
@@ -35,8 +35,9 @@
static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START];
+static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count);
static void page_privatize(uintptr_t pagenum);
-static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count);
+static void page_reshare(long segment_num, uintptr_t pagenum);
static void mutex_pages_lock(void);
static void mutex_pages_unlock(void);
diff --git a/c7/test/test_gcpage.py b/c7/test/test_gcpage.py
--- a/c7/test/test_gcpage.py
+++ b/c7/test/test_gcpage.py
@@ -202,7 +202,6 @@
#
self.start_transaction()
stm_major_collect()
- py.test.skip("XXX implement me")
assert lib._stm_total_allocated() == 5000 + LMO # shared again
def test_reshare_if_no_longer_modified_1(self):
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
