Author: Armin Rigo <[email protected]>
Branch: stmgc-c7
Changeset: r69887:e0337a4058f2
Date: 2014-03-12 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/e0337a4058f2/
Log: Reintroduce stm_perform_transaction() and atomic transactions.
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -66,11 +66,11 @@
@dont_look_inside
def increment_atomic():
- llop.stm_change_atomic(lltype.Signed, +1)
+ llop.stm_increment_atomic(lltype.Void)
@dont_look_inside
def decrement_atomic():
- llop.stm_change_atomic(lltype.Signed, -1)
+ llop.stm_decrement_atomic(lltype.Void)
@dont_look_inside
def is_atomic():
@@ -96,7 +96,7 @@
def before_external_call():
if we_are_translated():
# this tries to commit, or becomes inevitable if atomic
- llop.stm_commit_transaction(lltype.Void)
+ llop.stm_commit_if_not_atomic(lltype.Void)
before_external_call._dont_reach_me_in_del_ = True
before_external_call._transaction_break_ = True
@@ -104,7 +104,7 @@
def after_external_call():
if we_are_translated():
# starts a new transaction if we are not atomic already
- llop.stm_start_inevitable_transaction(lltype.Void)
+ llop.stm_start_inevitable_if_not_atomic(lltype.Void)
after_external_call._dont_reach_me_in_del_ = True
after_external_call._transaction_break_ = True
diff --git a/rpython/rtyper/lltypesystem/lloperation.py
b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -438,6 +438,7 @@
# see threadlocalref.py
'stm_threadlocal_get': LLOp(sideeffects=False),
'stm_threadlocal_set': LLOp(),
+ 'stm_perform_transaction':LLOp(canmallocgc=True),
## 'stm_allocate_nonmovable_int_adr': LLOp(sideeffects=False,
canmallocgc=True),
## 'stm_become_inevitable': LLOp(canmallocgc=True),
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -786,7 +786,8 @@
if node.forward_decl:
print >> f, node.forward_decl
elif node.name is not None:
- print >> f, '%s %s;' % (node.typetag, node.name)
+ if node.typetag != '':
+ print >> f, '%s %s;' % (node.typetag, node.name)
print >> f
for node in structdeflist:
for line in node.definition():
diff --git a/rpython/translator/stm/funcgen.py
b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -134,13 +134,11 @@
return '/* %s = */ STM_POP_ROOT_RET(stm_thread_local);' % (arg0,)
return 'STM_POP_ROOT(stm_thread_local, %s);' % (arg0,)
-def stm_commit_transaction(funcgen, op):
- return '{ int e = errno; stm_commit_transaction(); errno = e; }'
+def stm_commit_if_not_atomic(funcgen, op):
+ return 'pypy_stm_commit_if_not_atomic();'
-def stm_start_inevitable_transaction(funcgen, op):
- return ('{ int e = errno; '
- 'stm_start_inevitable_transaction(&stm_thread_local); '
- 'errno = e; }')
+def stm_start_inevitable_if_not_atomic(funcgen, op):
+ return 'pypy_stm_start_inevitable_if_not_atomic();'
def stm_enter_callback_call(funcgen, op):
result = funcgen.expr(op.result)
@@ -167,6 +165,12 @@
arg0 = funcgen.expr(op.args[0])
return 'stm_thread_local.thread_local_obj = (object_t *)%s;' % (arg0,)
+def stm_perform_transaction(funcgen, op):
+ arg0 = funcgen.expr(op.args[0])
+ arg1 = funcgen.expr(op.args[1])
+ return ('pypy_stm_perform_transaction((object_t *)%s, '
+ '(int(*)(object_t *, int))%s);' % (arg0, arg1))
+
##def stm_initialize(funcgen, op):
## return '''stm_initialize();
@@ -297,16 +301,6 @@
## result = funcgen.expr(op.result)
## return '%s = ((struct rpyobj_s*)%s)->tid;' % (result, arg0)
-##def stm_hash(funcgen, op):
-## arg0 = funcgen.expr(op.args[0])
-## result = funcgen.expr(op.result)
-## return '%s = stm_hash((gcptr)%s);' % (result, arg0)
-
-##def stm_id(funcgen, op):
-## arg0 = funcgen.expr(op.args[0])
-## result = funcgen.expr(op.result)
-## return '%s = stm_id((gcptr)%s);' % (result, arg0)
-
##def stm_change_atomic(funcgen, op):
## arg0 = funcgen.expr(op.args[0])
## return 'stm_atomic(%s);' % (arg0,)
@@ -315,20 +309,6 @@
## result = funcgen.expr(op.result)
## return '%s = stm_atomic(0);' % (result,)
-##def stm_threadlocal_get(funcgen, op):
-## result = funcgen.expr(op.result)
-## return '%s = (%s)stm_thread_local_obj;' % (
-## result, cdecl(funcgen.lltypename(op.result), ''))
-
-##def stm_threadlocal_set(funcgen, op):
-## arg0 = funcgen.expr(op.args[0])
-## return 'stm_thread_local_obj = (gcptr)%s;' % (arg0,)
-
-##def stm_perform_transaction(funcgen, op):
-## arg0 = funcgen.expr(op.args[0])
-## arg1 = funcgen.expr(op.args[1])
-## return 'stm_perform_transaction((gcptr)%s, %s);' % (arg0, arg1)
-
##def stm_enter_callback_call(funcgen, op):
## result = funcgen.expr(op.result)
## return '%s = stm_enter_callback_call();' % (result,)
diff --git a/rpython/translator/stm/src_stm/revision
b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-3ce4c20d80e7
+3f0d8773b90b
diff --git a/rpython/translator/stm/src_stm/stm/core.c
b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -138,6 +138,8 @@
void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf)
{
+ assert(!_stm_in_transaction(tl));
+
s_mutex_lock();
retry:
@@ -467,7 +469,7 @@
struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
/* throw away the content of the nursery */
- throw_away_nursery(pseg);
+ long bytes_in_nursery = throw_away_nursery(pseg);
/* reset all the modified objects (incl. re-adding GCFLAG_WRITE_BARRIER) */
reset_modified_from_other_segments(segment_num);
@@ -477,6 +479,7 @@
stm_thread_local_t *tl = pseg->pub.running_thread;
tl->shadowstack = pseg->shadowstack_at_start_of_transaction;
tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
+ tl->last_abort__bytes_in_nursery = bytes_in_nursery;
/* reset these lists to NULL too on abort */
LIST_FREE(pseg->objects_pointing_to_nursery);
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.c
b/rpython/translator/stm/src_stm/stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.c
@@ -4,11 +4,6 @@
/* ------------------------------------------------------------ */
-static int dprintfcolor(void)
-{
- return 31 + STM_SEGMENT->segment_num % 6;
-}
-
static int threadcolor_printf(const char *format, ...)
{
char buffer[2048];
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.h
b/rpython/translator/stm/src_stm/stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.h
@@ -8,7 +8,10 @@
#define dprintf(args) threadcolor_printf args
-static int dprintfcolor(void);
+static inline int dprintfcolor(void)
+{
+ return 31 + STM_SEGMENT->segment_num % 6;
+}
static int threadcolor_printf(const char *format, ...)
__attribute__((format (printf, 1, 2)));
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c
b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -216,15 +216,15 @@
_collect_now(item));
}
-static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
+static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
{
/* reset the nursery by zeroing it */
- size_t size;
+ size_t nursery_used;
char *realnursery;
realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start);
- size = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start;
- memset(realnursery, 0, size);
+ nursery_used = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start;
+ memset(realnursery, 0, nursery_used);
pseg->pub.nursery_current = (stm_char *)_stm_nursery_start;
@@ -251,6 +251,7 @@
}
tree_clear(pseg->nursery_objects_shadows);
+ return nursery_used;
}
#define MINOR_NOTHING_TO_DO(pseg) \
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h
b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -12,7 +12,7 @@
static void minor_collection(bool commit);
static void check_nursery_at_transaction_start(void);
-static void throw_away_nursery(struct stm_priv_segment_info_s *pseg);
+static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
static void major_do_minor_collections(void);
static inline bool must_abort(void) {
diff --git a/rpython/translator/stm/src_stm/stmgc.h
b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -58,7 +58,10 @@
the following raw region of memory is cleared. */
char *mem_clear_on_abort;
size_t mem_bytes_to_clear_on_abort;
- /* the next fields are handled automatically by the library */
+ /* after an abort, some details about the abort are stored there.
+ (these fields are not modified on a successful commit) */
+ long last_abort__bytes_in_nursery;
+ /* the next fields are handled internally by the library */
int associated_segment_num;
struct stm_thread_local_s *prev, *next;
} stm_thread_local_t;
@@ -246,6 +249,9 @@
if (STM_SEGMENT->jmpbuf_ptr != NULL)
_stm_become_inevitable(msg);
}
+static inline int stm_is_inevitable(void) {
+ return (STM_SEGMENT->jmpbuf_ptr == NULL);
+}
/* Forces a safe-point if needed. Normally not needed: this is
automatic if you call stm_allocate(). */
diff --git a/rpython/translator/stm/src_stm/stmgcintf.c
b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -4,6 +4,10 @@
__thread struct stm_thread_local_s stm_thread_local;
+/* 0 = not initialized; 1 = normal mode; 2 or more = atomic mode */
+__thread long pypy_stm_ready_atomic;
+__thread uintptr_t pypy_stm_nursery_low_fill_mark;
+
extern Signed pypy_stmcb_size_rounded_up(void*);
extern void pypy_stmcb_trace(void*, void(*)(void*));
@@ -28,7 +32,7 @@
#define LOW_FILL_MARK 400000
-stm_char *pypy_stm_nursery_low_fill_mark;
+static long pypy_transaction_length;
void pypy_stm_set_transaction_length(long percentage)
@@ -38,36 +42,124 @@
long low_fill_mark = LOW_FILL_MARK * percentage / 100;
if (low_fill_mark > NURSERY_SIZE / 2)
low_fill_mark = NURSERY_SIZE / 2;
- pypy_stm_nursery_low_fill_mark = ((stm_char *)_stm_nursery_start) +
- low_fill_mark;
+ pypy_transaction_length = low_fill_mark;
}
void pypy_stm_setup(void)
{
stm_setup();
stm_register_thread_local(&stm_thread_local);
+ pypy_stm_ready_atomic = 1;
pypy_stm_set_transaction_length(100);
- stm_start_inevitable_transaction(&stm_thread_local);
+ pypy_stm_start_inevitable_if_not_atomic();
}
long pypy_stm_enter_callback_call(void)
{
- long token = 0;
-
- if (stm_thread_local.shadowstack == NULL) {
+ if (pypy_stm_ready_atomic == 0) {
/* first time we see this thread */
- token = 1;
+ int e = errno;
stm_register_thread_local(&stm_thread_local);
+ errno = e;
+ pypy_stm_ready_atomic = 1;
+ pypy_stm_start_inevitable_if_not_atomic();
+ return 1;
}
- stm_start_inevitable_transaction(&stm_thread_local);
- return token;
+ else {
+ /* callback from C code, itself called from Python code */
+ pypy_stm_start_inevitable_if_not_atomic();
+ return 0;
+ }
}
void pypy_stm_leave_callback_call(long token)
{
- stm_commit_transaction();
if (token == 1) {
+ /* if we're returning into foreign C code that was not itself
+ called from Python code, then we're ignoring the atomic
+ status and committing anyway. */
+ int e = errno;
+ pypy_stm_ready_atomic = 1;
+ stm_commit_transaction();
+ pypy_stm_ready_atomic = 0;
stm_unregister_thread_local(&stm_thread_local);
- assert(stm_thread_local.shadowstack == NULL);
+ errno = e;
+ }
+ else {
+ pypy_stm_commit_if_not_atomic();
}
}
+
+void pypy_stm_perform_transaction(object_t *arg, int callback(object_t *, int))
+{ /* must save roots around this call */
+ stm_jmpbuf_t jmpbuf;
+ long volatile v_counter = 0;
+#ifndef NDEBUG
+ object_t **volatile old_shadowstack = stm_thread_local.shadowstack;
+#endif
+
+ STM_PUSH_ROOT(stm_thread_local, arg);
+ /*STM_PUSH_ROOT(END_MARKER_OFF); XXX redo this optimization */
+
+ while (1) {
+
+ if (pypy_stm_ready_atomic == 1) {
+ stm_commit_transaction();
+ STM_START_TRANSACTION(&stm_thread_local, jmpbuf);
+ }
+
+ /* After setjmp(), the local variables v_* are preserved because they
+ * are volatile. The other variables are only declared here. */
+ long counter, result;
+ counter = v_counter;
+ v_counter = counter + 1;
+
+ /* If counter==0, initialize 'pypy_stm_nursery_low_fill_mark'
+ from the configured length limit. If counter>0, we did an
+ abort, and we can now configure 'pypy_stm_nursery_low_fill_mark'
+ to a value slightly smaller than the value at last abort.
+ */
+ if (stm_is_inevitable()) {
+ pypy_stm_nursery_low_fill_mark = 0;
+ }
+ else {
+ long limit;
+ if (counter == 0) {
+ limit = pypy_transaction_length;
+ }
+ else {
+ limit = stm_thread_local.last_abort__bytes_in_nursery;
+ limit -= (limit >> 4);
+ }
+ pypy_stm_nursery_low_fill_mark = _stm_nursery_start + limit;
+ }
+
+ /* invoke the callback in the new transaction */
+ STM_POP_ROOT(stm_thread_local, arg);
+ assert(old_shadowstack == stm_thread_local.shadowstack);
+ STM_PUSH_ROOT(stm_thread_local, arg);
+ result = callback(arg, counter);
+ if (result <= 0)
+ break;
+ v_counter = 0;
+ }
+
+ if (STM_SEGMENT->jmpbuf_ptr == &jmpbuf) {
+ /* we can't leave this function leaving a non-inevitable
+ transaction whose jmpbuf points into this function
+ */
+ if (pypy_stm_ready_atomic == 1) {
+ stm_commit_transaction();
+ stm_start_inevitable_transaction(&stm_thread_local);
+ pypy_stm_nursery_low_fill_mark = 0;
+ }
+ else {
+ _stm_become_inevitable("perform_transaction left with atomic");
+ }
+ }
+
+ //gcptr x = stm_pop_root(); /* pop the END_MARKER */
+ //assert(x == END_MARKER_OFF || x == END_MARKER_ON);
+ STM_POP_ROOT_RET(stm_thread_local); /* pop the 'arg' */
+ assert(old_shadowstack == stm_thread_local.shadowstack);
+}
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h
b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -4,24 +4,55 @@
/* meant to be #included after src_stm/stmgc.h */
+#include <errno.h>
#include "stmgc.h"
#include "stm/atomic.h" /* for spin_loop() and write_fence() */
extern __thread struct stm_thread_local_s stm_thread_local;
-extern stm_char *pypy_stm_nursery_low_fill_mark;
+extern __thread long pypy_stm_ready_atomic;
+extern __thread uintptr_t pypy_stm_nursery_low_fill_mark;
void pypy_stm_setup(void);
void pypy_stm_setup_prebuilt(void); /* generated into stm_prebuilt.c */
+
+static inline void pypy_stm_commit_if_not_atomic(void) {
+ if (pypy_stm_ready_atomic == 1) {
+ int e = errno;
+ stm_commit_transaction();
+ errno = e;
+ }
+}
+static inline void pypy_stm_start_inevitable_if_not_atomic(void) {
+ if (pypy_stm_ready_atomic == 1) {
+ int e = errno;
+ stm_start_inevitable_transaction(&stm_thread_local);
+ pypy_stm_nursery_low_fill_mark = 0;
+ errno = e;
+ }
+}
+static inline void pypy_stm_increment_atomic(void) {
+ pypy_stm_ready_atomic++;
+}
+static inline void pypy_stm_decrement_atomic(void) {
+ if (--pypy_stm_ready_atomic == 0)
+ pypy_stm_ready_atomic = 1;
+}
+static inline long pypy_stm_get_atomic(void) {
+ return pypy_stm_ready_atomic - 1;
+}
long pypy_stm_enter_callback_call(void);
void pypy_stm_leave_callback_call(long);
void pypy_stm_set_transaction_length(long);
+void pypy_stm_perform_transaction(object_t *, int(object_t *, int));
static inline int pypy_stm_should_break_transaction(void)
{
/* we should break the current transaction if we have used more than
- some initial portion of the nursery, or if we are running inevitable */
- return (STM_SEGMENT->nursery_current >= pypy_stm_nursery_low_fill_mark ||
- STM_SEGMENT->jmpbuf_ptr == NULL);
+ some initial portion of the nursery, or if we are running inevitable
+ (in which case pypy_stm_nursery_low_fill_mark is set to 0)
+ */
+ uintptr_t current = (uintptr_t)STM_SEGMENT->nursery_current;
+ return current >= pypy_stm_nursery_low_fill_mark;
}
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
