Author: Armin Rigo <ar...@tunes.org>
Branch: c8-gil-like
Changeset: r1801:f344eac5014c
Date: 2015-06-11 11:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/f344eac5014c/
Log: in-progress
diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -347,7 +347,7 @@
objptr_t p;
- stm_start_transaction(&stm_thread_local);
+ stm_enter_transactional_zone(&stm_thread_local);
assert(td.num_roots >= td.num_roots_at_transaction_start);
td.num_roots = td.num_roots_at_transaction_start;
p = NULL;
@@ -367,12 +367,19 @@
long call_fork = (arg != NULL && *(long *)arg);
if (call_fork == 0) { /* common case */
- stm_commit_transaction();
td.num_roots_at_transaction_start = td.num_roots;
- if (get_rand(100) < 98) {
- stm_start_transaction(&stm_thread_local);
- } else {
- stm_start_inevitable_transaction(&stm_thread_local);
+ if (get_rand(100) < 50) {
+ stm_leave_transactional_zone(&stm_thread_local);
+ /* Nothing here; it's unlikely that a different thread
+ manages to steal the detached inev transaction.
+ Give them a little chance with a usleep(). */
+ fprintf(stderr, "sleep...\n");
+ usleep(1);
+ fprintf(stderr, "sleep done\n");
+ stm_enter_transactional_zone(&stm_thread_local);
+ }
+ else {
+ stm_force_transaction_break(&stm_thread_local);
}
td.num_roots = td.num_roots_at_transaction_start;
p = NULL;
@@ -401,16 +408,16 @@
}
}
push_roots();
- stm_commit_transaction();
+ stm_force_transaction_break(&stm_thread_local);
/* even out the shadow stack before leaveframe: */
- stm_start_inevitable_transaction(&stm_thread_local);
+ stm_become_inevitable(&stm_thread_local, "before leaveframe");
while (td.num_roots > 0) {
td.num_roots--;
objptr_t t;
STM_POP_ROOT(stm_thread_local, t);
}
- stm_commit_transaction();
+ stm_leave_transactional_zone(&stm_thread_local);
stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,15 +24,21 @@
#if defined(__i386__) || defined(__amd64__)
-# define HAVE_FULL_EXCHANGE_INSN
static inline void spin_loop(void) { asm("pause" : : : "memory"); }
static inline void write_fence(void) { asm("" : : : "memory"); }
+# define atomic_exchange(ptr, old, new) do { \
+ (old) = __sync_lock_test_and_set(ptr, new); \
+ } while (0)
#else
static inline void spin_loop(void) { asm("" : : : "memory"); }
static inline void write_fence(void) { __sync_synchronize(); }
+# define atomic_exchange(ptr, old, new) do { \
+ (old) = *(ptr); \
+ } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new)));
+
#endif
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -496,8 +496,8 @@
static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
{
- int detached = fetch_detached_transaction();
- if (detached >= 0) {
+ intptr_t detached = fetch_detached_transaction();
+ if (detached != 0) {
commit_fetched_detached_transaction(detached);
return;
}
@@ -509,7 +509,7 @@
usleep(10); /* XXXXXX */
detached = fetch_detached_transaction();
- if (detached >= 0) {
+ if (detached != 0) {
commit_fetched_detached_transaction(detached);
break;
}
@@ -1235,7 +1235,8 @@
list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
list_clear(STM_PSEGMENT->old_objects_with_cards_set);
list_clear(STM_PSEGMENT->large_overflow_objects);
- timing_event(tl, event);
+ if (tl != NULL)
+ timing_event(tl, event);
release_thread_segment(tl);
/* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1290,19 +1291,24 @@
assert(!_has_mutex());
assert(STM_PSEGMENT->safe_point == SP_RUNNING);
- //assert(STM_PSEGMENT->running_pthread == pthread_self());
- // ^^^ fails if detach.c commits a detached inevitable transaction
+ assert(STM_PSEGMENT->running_pthread == pthread_self());
dprintf(("> stm_commit_transaction()\n"));
minor_collection(1);
+ _core_commit_transaction();
+}
+
+static void _core_commit_transaction(void)
+{
push_large_overflow_objects_to_other_segments();
/* push before validate. otherwise they are reachable too early */
bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
_validate_and_add_to_commit_log();
- stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+ if (!was_inev)
+ stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
/* XXX do we still need a s_mutex_lock() section here? */
s_mutex_lock();
@@ -1343,7 +1349,8 @@
/* between transactions, call finalizers. this will execute
a transaction itself */
- invoke_general_finalizers(tl);
+ if (tl != NULL)
+ invoke_general_finalizers(tl);
}
static void reset_modified_from_backup_copies(int segment_num)
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -300,6 +300,7 @@
static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
static bool _stm_validate(void);
+static void _core_commit_transaction(void);
static inline bool was_read_remote(char *base, object_t *obj)
{
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -2,21 +2,31 @@
# error "must be compiled via stmgc.c"
#endif
+/* Idea: if stm_leave_transactional_zone() is quickly followed by
+ stm_enter_transactional_zone() in the same thread, then we should
+ simply try to have one inevitable transaction that does both sides.
+ This is useful if there are many such small interruptions.
-/* _stm_detached_inevitable_segnum is:
+ stm_leave_transactional_zone() tries to make sure the transaction
+ is inevitable, and then sticks the current 'stm_thread_local_t *'
+ into _stm_detached_inevitable_from_thread.
+ stm_enter_transactional_zone() has a fast-path if the same
+ 'stm_thread_local_t *' is still there.
- - -1: there is no inevitable transaction, or it is not detached
+ If a different thread grabs it, it atomically replaces the value in
+ _stm_detached_inevitable_from_thread with -1, commits it (this part
+ involves reading for example the shadowstack of the thread that
+ originally detached), and at the point where we know the original
+ stm_thread_local_t is no longer relevant, we reset
+ _stm_detached_inevitable_from_thread to 0.
+*/
- - in range(1, NB_SEGMENTS): an inevitable transaction belongs to
- the segment and was detached. It might concurrently be
- reattached at any time, with an XCHG (__sync_lock_test_and_set).
-*/
-volatile int _stm_detached_inevitable_seg_num;
+volatile intptr_t _stm_detached_inevitable_from_thread;
static void setup_detach(void)
{
- _stm_detached_inevitable_seg_num = -1;
+ _stm_detached_inevitable_from_thread = 0;
}
@@ -26,50 +36,108 @@
/* did it work? */
if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { /* yes */
- _stm_detach_inevitable_transaction(STM_SEGMENT->running_thread);
+ dprintf(("leave_noninevitable_transactional_zone: now inevitable\n"));
+ stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+ _stm_detach_inevitable_transaction(tl);
}
else { /* no */
+ dprintf(("leave_noninevitable_transactional_zone: commit\n"));
_stm_commit_transaction();
}
}
-void _stm_reattach_transaction(int old, stm_thread_local_t *tl)
+static void commit_external_inevitable_transaction(void)
{
- if (old == -1) {
- /* there was no detached inevitable transaction */
- _stm_start_transaction(tl);
+ assert(!_has_mutex());
+ assert(STM_PSEGMENT->safe_point == SP_RUNNING);
+ assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); /* can't abort */
+
+ exec_local_finalizers();
+ minor_collection(1);
+
+ /* from this point on, unlink the original 'stm_thread_local_t *'
+ from its segment. Better do it as soon as possible, because
+ other threads might be spin-looping, waiting for the -1 to
+ disappear. XXX could be done even earlier, as soon as we have
+ read the shadowstack inside the minor collection. */
+ STM_SEGMENT->running_thread = NULL;
+ write_fence();
+ assert(_stm_detached_inevitable_from_thread == -1);
+ _stm_detached_inevitable_from_thread = 0;
+
+ _core_commit_transaction();
+}
+
+void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl)
+{
+ restart:
+ if (old != 0) {
+ if (old == -1) {
+ /* busy-loop: wait until _stm_detached_inevitable_from_thread
+ is reset to a value different from -1 */
+ while (_stm_detached_inevitable_from_thread == -1)
+ spin_loop();
+
+ /* then retry */
+ atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
+ goto restart;
+ }
+
+ stm_thread_local_t *old_tl = (stm_thread_local_t *)old;
+ int remote_seg_num = old_tl->last_associated_segment_num;
+ dprintf(("reattach_transaction: commit detached from seg %d\n",
+ remote_seg_num));
+
+ ensure_gs_register(remote_seg_num);
+ commit_external_inevitable_transaction();
}
else {
- /* We took over the inevitable transaction originally detached
- from a different segment. We have to fix the %gs register if
- it is incorrect.
- */
- tl->last_associated_segment_num = old;
- ensure_gs_register(old);
- assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
- STM_SEGMENT->running_thread = tl;
-
- stm_safe_point();
+ assert(_stm_detached_inevitable_from_thread == -1);
+ _stm_detached_inevitable_from_thread = 0;
}
+ dprintf(("reattach_transaction: start a new transaction\n"));
+ _stm_start_transaction(tl);
}
void stm_force_transaction_break(stm_thread_local_t *tl)
{
+ dprintf(("> stm_force_transaction_break()\n"));
assert(STM_SEGMENT->running_thread == tl);
_stm_commit_transaction();
_stm_start_transaction(tl);
}
-static int fetch_detached_transaction(void)
+static intptr_t fetch_detached_transaction(void)
{
- int cur = _stm_detached_inevitable_seg_num;
- if (cur != -1)
- cur = __sync_lock_test_and_set( /* XCHG */
- &_stm_detached_inevitable_seg_num, -1);
+ intptr_t cur;
+ restart:
+ cur = _stm_detached_inevitable_from_thread;
+ if (cur == 0) { /* fast-path */
+ return 0; /* _stm_detached_inevitable_from_thread not changed */
+ }
+ if (cur != -1) {
+ atomic_exchange(&_stm_detached_inevitable_from_thread, cur, -1);
+ if (cur == 0) {
+ /* found 0, so change from -1 to 0 again and return */
+ _stm_detached_inevitable_from_thread = 0;
+ return 0;
+ }
+ }
+ if (cur == -1) {
+ /* busy-loop: wait until _stm_detached_inevitable_from_thread
+ is reset to a value different from -1 */
+ while (_stm_detached_inevitable_from_thread == -1)
+ spin_loop();
+ goto restart;
+ }
+ /* this is the only case where we grabbed a detached transaction.
+ _stm_detached_inevitable_from_thread is still -1, until
+ commit_fetched_detached_transaction() is called. */
+ assert(_stm_detached_inevitable_from_thread == -1);
return cur;
}
-static void commit_fetched_detached_transaction(int segnum)
+static void commit_fetched_detached_transaction(intptr_t old)
{
/* Here, 'seg_num' is the segment that contains the detached
inevitable transaction from fetch_detached_transaction(),
@@ -79,13 +147,33 @@
transaction. This should guarantee there are not race
conditions.
*/
+ int segnum = ((stm_thread_local_t *)old)->last_associated_segment_num;
+ dprintf(("commit_fetched_detached_transaction from seg %d\n", segnum));
assert(segnum > 0);
int mysegnum = STM_SEGMENT->segment_num;
ensure_gs_register(segnum);
-
- assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
- _stm_commit_transaction(); /* can't abort */
-
+ commit_external_inevitable_transaction();
ensure_gs_register(mysegnum);
}
+
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl)
+{
+ intptr_t old;
+ restart:
+ old = _stm_detached_inevitable_from_thread;
+ if (old == (intptr_t)tl) {
+ if
(!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
+ old, -1))
+ goto restart;
+ commit_fetched_detached_transaction(old);
+ return;
+ }
+ if (old == -1) {
+ /* busy-loop: wait until _stm_detached_inevitable_from_thread
+ is reset to a value different from -1 */
+ while (_stm_detached_inevitable_from_thread == -1)
+ spin_loop();
+ goto restart;
+ }
+}
diff --git a/c8/stm/detach.h b/c8/stm/detach.h
--- a/c8/stm/detach.h
+++ b/c8/stm/detach.h
@@ -1,4 +1,5 @@
static void setup_detach(void);
-static int fetch_detached_transaction(void);
-static void commit_fetched_detached_transaction(int segnum);
+static intptr_t fetch_detached_transaction(void);
+static void commit_fetched_detached_transaction(intptr_t old);
+static void commit_detached_transaction_if_from(stm_thread_local_t *tl);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -264,6 +264,8 @@
void stm_unregister_thread_local(stm_thread_local_t *tl)
{
+ commit_detached_transaction_if_from(tl);
+
s_mutex_lock();
assert(tl->prev != NULL);
assert(tl->next != NULL);
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -215,10 +215,12 @@
num = (num+1) % (NB_SEGMENTS-1);
if (sync_ctl.in_use1[num+1] == 0) {
/* we're getting 'num', a different number. */
- dprintf(("acquired different segment: %d->%d\n",
- tl->last_associated_segment_num, num+1));
+ int old_num = tl->last_associated_segment_num;
+ dprintf(("acquired different segment: %d->%d\n", old_num, num+1));
tl->last_associated_segment_num = num+1;
set_gs_register(get_segment_base(num+1));
+ dprintf((" %d->%d\n", old_num, num+1));
+ (void)old_num;
goto got_num;
}
}
@@ -245,18 +247,22 @@
static void release_thread_segment(stm_thread_local_t *tl)
{
+ int segnum;
assert(_has_mutex());
cond_signal(C_SEGMENT_FREE);
assert(STM_SEGMENT->running_thread == tl);
- assert(tl->last_associated_segment_num == STM_SEGMENT->segment_num);
- assert(in_transaction(tl));
- STM_SEGMENT->running_thread = NULL;
- assert(!in_transaction(tl));
+ segnum = STM_SEGMENT->segment_num;
+ if (tl != NULL) {
+ assert(tl->last_associated_segment_num == segnum);
+ assert(in_transaction(tl));
+ STM_SEGMENT->running_thread = NULL;
+ assert(!in_transaction(tl));
+ }
- assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
- sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
+ assert(sync_ctl.in_use1[segnum] == 1);
+ sync_ctl.in_use1[segnum] = 0;
}
__attribute__((unused))
@@ -414,8 +420,8 @@
Wait until all other threads are suspended. */
while (count_other_threads_sp_running() > 0) {
- int detached = fetch_detached_transaction();
- if (detached >= 0) {
+ intptr_t detached = fetch_detached_transaction();
+ if (detached != 0) {
remove_requests_for_safe_point(); /* => C_REQUEST_REMOVED */
commit_fetched_detached_transaction(detached);
goto restart;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -84,16 +84,16 @@
object_t *_stm_allocate_slowpath(ssize_t);
object_t *_stm_allocate_external(ssize_t);
-extern volatile int _stm_detached_inevitable_seg_num;
+extern volatile intptr_t _stm_detached_inevitable_from_thread;
long _stm_start_transaction(stm_thread_local_t *tl);
void _stm_commit_transaction(void);
void _stm_leave_noninevitable_transactional_zone(void);
-#define _stm_detach_inevitable_transaction(tl) do { \
- write_fence(); \
- assert((tl)->last_associated_segment_num == STM_SEGMENT->segment_num); \
- _stm_detached_inevitable_seg_num = STM_SEGMENT->segment_num; \
+#define _stm_detach_inevitable_transaction(tl) do { \
+ write_fence(); \
+ assert(_stm_detached_inevitable_from_thread == 0); \
+ _stm_detached_inevitable_from_thread = (intptr_t)(tl); \
} while (0)
-void _stm_reattach_transaction(int old, stm_thread_local_t *tl);
+void _stm_reattach_transaction(intptr_t old, stm_thread_local_t *tl);
void _stm_become_inevitable(const char*);
void _stm_collectable_safe_point(void);
@@ -421,12 +421,15 @@
transactions.
*/
static inline void stm_enter_transactional_zone(stm_thread_local_t *tl) {
- int old = __sync_lock_test_and_set( /* XCHG */
- &_stm_detached_inevitable_seg_num, -1);
- if (old == tl->last_associated_segment_num)
- STM_SEGMENT->running_thread = tl;
- else
+ intptr_t old;
+ atomic_exchange(&_stm_detached_inevitable_from_thread, old, -1);
+ if (old == (intptr_t)tl) {
+ _stm_detached_inevitable_from_thread = 0;
+ }
+ else {
_stm_reattach_transaction(old, tl);
+ assert(_stm_detached_inevitable_from_thread == 0);
+ }
}
static inline void stm_leave_transactional_zone(stm_thread_local_t *tl) {
assert(STM_SEGMENT->running_thread == tl);
@@ -458,7 +461,7 @@
if (!stm_is_inevitable())
_stm_become_inevitable(msg);
/* now, we're running the inevitable transaction, so: */
- assert(_stm_detached_inevitable_seg_num == -1);
+ assert(_stm_detached_inevitable_from_thread == 0);
}
/* Forces a safe-point if needed. Normally not needed: this is
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
