Hello,
Some time ago I ported jemalloc to work on old systems still using LinuxThreads. See this thread: http://www.canonware.com/pipermail/jemalloc-discuss/2013-October/000646.html It seems that with LinuxThreads the fork() implementation will call free during fork, between jemalloc_prefork and jemalloc_postfork_child. This can sometimes (very rarely) hang in a call to system() with a stack like the following: #0 0x0fdd660c in __pthread_sigsuspend () from /lib/libpthread.so.0 #1 0x0fdd6344 in __pthread_wait_for_restart_signal () from /lib/libpthread.so.0 #2 0x0fdd805c in __pthread_alt_lock () from /lib/libpthread.so.0 #3 0x0fdd4c74 in pthread_mutex_lock () from /lib/libpthread.so.0 #4 0x0ff2a480 in pthread_mutex_lock () from /lib/libc.so.6 #5 0x0ffc5bf0 in malloc_mutex_lock (tbin=0x3082c020, binind=0, rem=0, tcache=0x3082c000) at ../../src/jemalloc-3.0.0/include/jemalloc/internal/mutex.h:77 #6 tcache_bin_flush_small (tbin=0x3082c020, binind=0, rem=0, tcache=0x3082c000) at ../../src/jemalloc-3.0.0/src/tcache.c:106 #7 0x0ffc64c0 in tcache_event_hard (tcache=0x3082c000) at ../../src/jemalloc-3.0.0/src/tcache.c:39 #8 0x0ffa545c in tcache_event (ptr=0x3081e000) at ../../src/jemalloc-3.0.0/include/jemalloc/internal/tcache.h:271 #9 tcache_dalloc_large (ptr=0x3081e000) at ../../src/jemalloc-3.0.0/include/jemalloc/internal/tcache.h:435 #10 arena_dalloc (ptr=0x3081e000) at ../../src/jemalloc-3.0.0/include/jemalloc/internal/arena.h:966 #11 idalloc (ptr=0x3081e000) at include/jemalloc/internal/jemalloc_internal.h:840 #12 iqalloc (ptr=0x3081e000) at include/jemalloc/internal/jemalloc_internal.h:852 #13 free (ptr=0x3081e000) at ../../src/jemalloc-3.0.0/src/jemalloc.c:1219 #14 0x0fdd6174 in __pthread_reset_main_thread () from /lib/libpthread.so.0 #15 0x0fdd5288 in __pthread_fork () from /lib/libpthread.so.0 #16 0x0feeadc4 in fork () from /lib/libc.so.6 #17 0x0fe82eb0 in do_system () from /lib/libc.so.6 #18 0x0fe830c8 in system () from /lib/libc.so.6 I am not familiar with how jemalloc works internally but it seems that sometimes tcache_event will trigger some sort of GC. Sometimes (very rarely) this attempts to take a lock which is already taken inside jemalloc_prefork. This hangs because locks are not recursive by default. I was able to reproduce this in a standalone program. The issue seems to go away if I avoid the GC as in the attached patch. It seems like a horrible evil hack. * Am I missing anything? Are there any other platforms with this issue? * ?Can you see something else going wrong because of free-during-fork? I could patch jemalloc to leak that memory. I only really care about system(), not other uses of fork(). * Can you think of a cleaner solution? The patch is on top of 3.0.0 so it won't apply cleanly. I tried to apply commits 20f1fc95adb35ea63dc61f47f2b0ffbd37d39f32<https://github.com/jemalloc/jemalloc/commit/20f1fc95adb35ea63dc61f47f2b0ffbd37d39f32> and b5225928fe106a7d809bd34e849abcd6941e93c7<https://github.com/jemalloc/jemalloc/commit/b5225928fe106a7d809bd34e849abcd6941e93c7> but they did not help me. Regards, Leonard
commit 5fe93225a5e33b5b91e305338c470c4e35f6949e Author: Crestez Dan Leonard <[email protected]> Date: Wed May 14 00:02:56 2014 +0300 Track if inside a fork and prevent tcache_event_hard GC diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 268cd14..b5a36ba 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -517,6 +517,7 @@ extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ */ extern arena_t **arenas; extern unsigned narenas; +extern bool malloc_forking; arena_t *arenas_extend(unsigned ind); void arenas_cleanup(void *arg); diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 38d735c..893280e 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -265,6 +265,9 @@ tcache_event(tcache_t *tcache) if (TCACHE_GC_INCR == 0) return; + if (malloc_forking) + return; + tcache->ev_cnt++; assert(tcache->ev_cnt <= TCACHE_GC_INCR); if (tcache->ev_cnt == TCACHE_GC_INCR) diff --git a/src/jemalloc.c b/src/jemalloc.c index 7bc69ac..39f5249 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -38,6 +38,9 @@ unsigned narenas; /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; +/* Set to true between jemalloc_prefork and jemalloc_postfork_* */ +bool malloc_forking = false; + #ifdef JEMALLOC_THREADED_INIT /* Used to let the initializing thread recursively allocate. */ # define NO_INITIALIZER ((unsigned long)0) @@ -1632,6 +1635,7 @@ _malloc_prefork(void) return; #endif assert(malloc_initialized); + assert(!malloc_forking); /* Acquire all mutexes in a safe order. */ malloc_mutex_prefork(&arenas_lock); @@ -1642,6 +1646,8 @@ _malloc_prefork(void) base_prefork(); huge_prefork(); chunk_dss_prefork(); + + malloc_forking = true; } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -1659,6 +1665,7 @@ _malloc_postfork(void) return; #endif assert(malloc_initialized); + assert(malloc_forking); /* Release all mutexes, now that fork() has completed. */ chunk_dss_postfork_parent(); @@ -1669,6 +1676,7 @@ _malloc_postfork(void) arena_postfork_parent(arenas[i]); } malloc_mutex_postfork_parent(&arenas_lock); + malloc_forking = false; } void @@ -1677,6 +1685,7 @@ jemalloc_postfork_child(void) unsigned i; assert(malloc_initialized); + assert(malloc_forking); /* Release all mutexes, now that fork() has completed. */ chunk_dss_postfork_child(); @@ -1687,6 +1696,7 @@ jemalloc_postfork_child(void) arena_postfork_child(arenas[i]); } malloc_mutex_postfork_child(&arenas_lock); + malloc_forking = false; } /******************************************************************************/
_______________________________________________ jemalloc-discuss mailing list [email protected] http://www.canonware.com/mailman/listinfo/jemalloc-discuss
