Module Name: src
Committed By: ad
Date: Fri Oct 13 19:30:28 UTC 2023
Modified Files:
src/lib/libc/stdlib: jemalloc.c
Log Message:
Minor changes to jemalloc100 (the old one that only vax etc currently uses).
- Don't use TLS nor pretend to hash out arenas to reduce lock contention,
because NetBSD uses thr_curcpu() to choose arena (i.e. per-CPU arenas).
- In a single threaded process, don't prepare "ncpus" worth of arenas,
allocate only one.
- Use getpagesize() since it caches the return.
- Sprinkle branch hints.
- Make MALLOC_TRACE and MALLOC_DEBUG work again.
To generate a diff of this commit:
cvs rdiff -u -r1.56 -r1.57 src/lib/libc/stdlib/jemalloc.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/libc/stdlib/jemalloc.c
diff -u src/lib/libc/stdlib/jemalloc.c:1.56 src/lib/libc/stdlib/jemalloc.c:1.57
--- src/lib/libc/stdlib/jemalloc.c:1.56 Sun May 7 12:41:47 2023
+++ src/lib/libc/stdlib/jemalloc.c Fri Oct 13 19:30:28 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: jemalloc.c,v 1.56 2023/05/07 12:41:47 skrll Exp $ */
+/* $NetBSD: jemalloc.c,v 1.57 2023/10/13 19:30:28 ad Exp $ */
/*-
* Copyright (C) 2006,2007 Jason Evans <[email protected]>.
@@ -97,13 +97,6 @@
/* LINTLIBRARY */
-#ifdef __NetBSD__
-# define xutrace(a, b) utrace("malloc", (a), (b))
-# define __DECONST(x, y) ((x)__UNCONST(y))
-#else
-# define xutrace(a, b) utrace((a), (b))
-#endif /* __NetBSD__ */
-
/*
* MALLOC_PRODUCTION disables assertions and statistics gathering. It also
* defaults the A and J runtime options to off. These settings are appropriate
@@ -117,21 +110,11 @@
#include <sys/cdefs.h>
/* __FBSDID("$FreeBSD: src/lib/libc/stdlib/malloc.c,v 1.147 2007/06/15 22:00:16 jasone Exp $"); */
-__RCSID("$NetBSD: jemalloc.c,v 1.56 2023/05/07 12:41:47 skrll Exp $");
+__RCSID("$NetBSD: jemalloc.c,v 1.57 2023/10/13 19:30:28 ad Exp $");
-#ifdef __FreeBSD__
-#include "libc_private.h"
-#ifdef MALLOC_DEBUG
-# define _LOCK_DEBUG
-#endif
-#include "spinlock.h"
-#endif
#include "namespace.h"
#include <sys/mman.h>
#include <sys/param.h>
-#ifdef __FreeBSD__
-#include <sys/stddef.h>
-#endif
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysctl.h>
@@ -139,12 +122,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
#include <sys/uio.h>
#include <sys/ktrace.h> /* Must come after several other sys/ includes. */
-#ifdef __FreeBSD__
-#include <machine/atomic.h>
-#include <machine/cpufunc.h>
-#include <machine/vmparam.h>
-#endif
-
#include <errno.h>
#include <limits.h>
#include <pthread.h>
@@ -158,17 +135,10 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
#include <strings.h>
#include <unistd.h>
-#ifdef __NetBSD__
-# include <reentrant.h>
-# include "extern.h"
+#include <reentrant.h>
+#include "extern.h"
#define STRERROR_R(a, b, c) strerror_r_ss(a, b, c);
-#endif
-
-#ifdef __FreeBSD__
-#define STRERROR_R(a, b, c) strerror_r(a, b, c);
-#include "un-namespace.h"
-#endif
/* MALLOC_STATS enables statistics calculation. */
#ifndef MALLOC_PRODUCTION
@@ -260,7 +230,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
# define QUANTUM_2POW_MIN 4
# define SIZEOF_PTR_2POW 2
# define USE_BRK
-# define NO_TLS
#endif
#ifdef __sh__
# define QUANTUM_2POW_MIN 4
@@ -271,9 +240,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
# define QUANTUM_2POW_MIN 4
# define SIZEOF_PTR_2POW 2
# define USE_BRK
-# ifdef __mc68010__
-# define NO_TLS
-# endif
#endif
#if defined(__mips__)
# ifdef _LP64
@@ -284,9 +250,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
# endif
# define QUANTUM_2POW_MIN 4
# define USE_BRK
-# if defined(__mips__)
-# define NO_TLS
-# endif
#endif
#if defined(__riscv__)
# ifdef _LP64
@@ -297,7 +260,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
# endif
# define QUANTUM_2POW_MIN 4
# define USE_BRK
-# define NO_TLS
#endif
#ifdef __hppa__
# define QUANTUM_2POW_MIN 4
@@ -367,21 +329,6 @@ __RCSID("$NetBSD: jemalloc.c,v 1.56 2023
/******************************************************************************/
-#ifdef __FreeBSD__
-/*
- * Mutexes based on spinlocks. We can't use normal pthread mutexes, because
- * they require malloc()ed memory.
- */
-typedef struct {
- spinlock_t lock;
-} malloc_mutex_t;
-
-/* Set to true once the allocator has been initialized. */
-static bool malloc_initialized = false;
-
-/* Used to avoid initialization races. */
-static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
-#else
#define malloc_mutex_t mutex_t
/* Set to true once the allocator has been initialized. */
@@ -391,7 +338,6 @@ static bool malloc_initialized = false;
/* Used to avoid initialization races. */
static mutex_t init_lock = MUTEX_INITIALIZER;
#endif
-#endif
/******************************************************************************/
/*
@@ -774,71 +720,10 @@ static size_t base_mapped;
* arenas array are necessarily used; arenas are created lazily as needed.
*/
static arena_t **arenas;
-static unsigned narenas;
-static unsigned next_arena;
#ifdef _REENTRANT
static malloc_mutex_t arenas_mtx; /* Protects arenas initialization. */
#endif
-/*
- * Map of pthread_self() --> arenas[???], used for selecting an arena to use
- * for allocations.
- */
-#ifndef NO_TLS
-static __attribute__((tls_model("initial-exec")))
-__thread arena_t **arenas_map;
-#else
-static arena_t **arenas_map;
-#endif
-
-#if !defined(NO_TLS) || !defined(_REENTRANT)
-# define get_arenas_map() (arenas_map)
-# define set_arenas_map(x) (arenas_map = x)
-#else /* NO_TLS && _REENTRANT */
-
-static thread_key_t arenas_map_key = -1;
-
-static inline arena_t **
-get_arenas_map(void)
-{
- if (!__isthreaded)
- return arenas_map;
-
- if (arenas_map_key == -1) {
- (void)thr_keycreate(&arenas_map_key, NULL);
- if (arenas_map != NULL) {
- thr_setspecific(arenas_map_key, arenas_map);
- arenas_map = NULL;
- }
- }
-
- return thr_getspecific(arenas_map_key);
-}
-
-static __inline void
-set_arenas_map(arena_t **a)
-{
- if (!__isthreaded) {
- arenas_map = a;
- return;
- }
-
- if (arenas_map_key == -1) {
-#ifndef NO_TLS
- (void)thr_keycreate(&arenas_map_key, NULL);
-#endif
- if (arenas_map != NULL) {
- _DIAGASSERT(arenas_map == a);
- arenas_map = NULL;
- }
- }
-
-#ifndef NO_TLS
- thr_setspecific(arenas_map_key, a);
-#endif
-}
-#endif /* NO_TLS && _REENTRANT */
-
#ifdef MALLOC_STATS
/* Chunk statistics. */
static chunk_stats_t stats_chunks;
@@ -866,7 +751,6 @@ static bool opt_utrace = false;
static bool opt_sysv = false;
static bool opt_xmalloc = false;
static bool opt_zero = false;
-static int32_t opt_narenas_lshift = 0;
typedef struct {
void *p;
@@ -874,13 +758,18 @@ typedef struct {
void *r;
} malloc_utrace_t;
+/* Sprinkle branch hints for the compiler and CPU. */
+#define OPT(a) __predict_false(opt_##a)
+#define NOT_OPT(a) __predict_true(!opt_##a)
+
+/* Trace malloc/free for ktrace/kdump. */
#define UTRACE(a, b, c) \
- if (opt_utrace) { \
+ if (OPT(utrace)) { \
malloc_utrace_t ut; \
ut.p = a; \
ut.s = b; \
ut.r = c; \
- xutrace(&ut, sizeof(ut)); \
+ utrace("malloc", &ut, sizeof(ut)); \
}
/******************************************************************************/
@@ -920,8 +809,8 @@ static void *arena_palloc(arena_t *arena
static size_t arena_salloc(const void *ptr);
static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
static void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-static bool arena_new(arena_t *arena);
-static arena_t *arenas_extend(unsigned ind);
+static void arena_new(arena_t *arena);
+static arena_t *arenas_extend(void);
static void *huge_malloc(size_t size);
static void *huge_palloc(size_t alignment, size_t size);
static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
@@ -1216,8 +1105,8 @@ base_chunk_node_dealloc(chunk_node_t *no
static void
stats_print(arena_t *arena)
{
- unsigned i;
- int gap_start;
+ const unsigned minusone = (unsigned)-1;
+ unsigned i, gap_start;
malloc_printf(
" allocated/mapped nmalloc ndalloc\n");
@@ -1236,12 +1125,12 @@ stats_print(arena_t *arena)
malloc_printf("bins: bin size regs pgs requests newruns"
" reruns maxruns curruns\n");
- for (i = 0, gap_start = -1; i < ntbins + nqbins + nsbins; i++) {
+ for (i = 0, gap_start = minusone; i < ntbins + nqbins + nsbins; i++) {
if (arena->bins[i].stats.nrequests == 0) {
- if (gap_start == -1)
+ if (gap_start == minusone)
gap_start = i;
} else {
- if (gap_start != -1) {
+ if (gap_start != minusone) {
if (i > gap_start + 1) {
/* Gap of more than one size class. */
malloc_printf("[%u..%u]\n",
@@ -1250,7 +1139,7 @@ stats_print(arena_t *arena)
/* Gap of one size class. */
malloc_printf("[%u]\n", gap_start);
}
- gap_start = -1;
+ gap_start = minusone;
}
malloc_printf(
"%13u %1s %4u %4u %3u %9llu %9llu"
@@ -1267,7 +1156,7 @@ stats_print(arena_t *arena)
arena->bins[i].stats.curruns);
}
}
- if (gap_start != -1) {
+ if (gap_start != minusone) {
if (i > gap_start + 1) {
/* Gap of more than one size class. */
malloc_printf("[%u..%u]\n", gap_start, i - 1);
@@ -1330,7 +1219,7 @@ pages_map_align(void *addr, size_t size,
STRERROR_R(errno, buf, sizeof(buf));
_malloc_message(getprogname(),
": (malloc) Error in munmap(): ", buf, "\n");
- if (opt_abort)
+ if (OPT(abort))
abort();
}
ret = NULL;
@@ -1358,7 +1247,7 @@ pages_unmap(void *addr, size_t size)
STRERROR_R(errno, buf, sizeof(buf));
_malloc_message(getprogname(),
": (malloc) Error in munmap(): ", buf, "\n");
- if (opt_abort)
+ if (OPT(abort))
abort();
}
}
@@ -1595,54 +1484,34 @@ chunk_dealloc(void *chunk, size_t size)
*/
/*
- * Choose an arena based on a per-thread and (optimistically) per-CPU value.
- *
- * We maintain at least one block of arenas. Usually there are more.
- * The blocks are $ncpu arenas in size. Whole blocks are 'hashed'
- * amongst threads. To accomplish this, next_arena advances only in
- * ncpu steps.
+ * Choose a per-CPU arena.
*/
static __noinline arena_t *
choose_arena_hard(void)
{
- unsigned i, curcpu;
- arena_t **map;
- /* Initialize the current block of arenas and advance to next. */
+ assert(arenas[0] != NULL);
+
malloc_mutex_lock(&arenas_mtx);
- assert(next_arena % ncpus == 0);
- assert(narenas % ncpus == 0);
- map = &arenas[next_arena];
- set_arenas_map(map);
- for (i = 0; i < ncpus; i++) {
- if (arenas[next_arena] == NULL)
- arenas_extend(next_arena);
- next_arena = (next_arena + 1) % narenas;
- }
+ for (unsigned i = 1; i < ncpus; i++)
+ if (arenas[i] == NULL)
+ arenas[i] = arenas_extend();
malloc_mutex_unlock(&arenas_mtx);
- /*
- * If we were unable to allocate an arena above, then default to
- * the first arena, which is always present.
- */
- curcpu = thr_curcpu();
- if (map[curcpu] != NULL)
- return map[curcpu];
- return arenas[0];
+ return arenas[thr_curcpu()];
}
static inline arena_t *
choose_arena(void)
{
- unsigned curcpu;
- arena_t **map;
+ arena_t *arena;
- map = get_arenas_map();
- curcpu = thr_curcpu();
- if (__predict_true(map != NULL && map[curcpu] != NULL))
- return map[curcpu];
+ /* NB: when libpthread is absent, thr_curcpu() always returns zero. */
+ arena = arenas[thr_curcpu()];
+ if (__predict_true(arena != NULL))
+ return arena;
- return choose_arena_hard();
+ return choose_arena_hard();
}
static inline int
@@ -1940,7 +1809,7 @@ arena_chunk_dealloc(arena_t *arena, aren
*/
RB_REMOVE(arena_chunk_tree_s, &chunk->arena->chunks, chunk);
- if (opt_hint == false) {
+ if (NOT_OPT(hint)) {
if (arena->spare != NULL) {
chunk_dealloc((void *)arena->spare, chunksize);
#ifdef MALLOC_STATS
@@ -2082,7 +1951,7 @@ arena_run_dalloc(arena_t *arena, arena_r
* Tell the kernel that we don't need the data in this run, but only if
* requested via runtime configuration.
*/
- if (opt_hint)
+ if (OPT(hint))
madvise(run, size, MADV_FREE);
/* Try to coalesce with neighboring runs. */
@@ -2373,9 +2242,9 @@ arena_malloc(arena_t *arena, size_t size
malloc_mutex_unlock(&arena->mtx);
- if (opt_junk)
+ if (OPT(junk))
memset(ret, 0xa5, size);
- else if (opt_zero)
+ else if (OPT(zero))
memset(ret, 0, size);
return (ret);
}
@@ -2473,9 +2342,9 @@ arena_palloc(arena_t *arena, size_t alig
#endif
malloc_mutex_unlock(&arena->mtx);
- if (opt_junk)
+ if (OPT(junk))
memset(ret, 0xa5, size);
- else if (opt_zero)
+ else if (OPT(zero))
memset(ret, 0, size);
return (ret);
}
@@ -2558,9 +2427,9 @@ arena_ralloc(void *ptr, size_t size, siz
idalloc(ptr);
return (ret);
IN_PLACE:
- if (opt_junk && size < oldsize)
+ if (OPT(junk) && size < oldsize)
memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
- else if (opt_zero && size > oldsize)
+ else if (OPT(zero) && size > oldsize)
memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
return (ptr);
}
@@ -2596,7 +2465,7 @@ arena_dalloc(arena_t *arena, arena_chunk
bin = run->bin;
size = bin->reg_size;
- if (opt_junk)
+ if (OPT(junk))
memset(ptr, 0x5a, size);
malloc_mutex_lock(&arena->mtx);
@@ -2652,7 +2521,7 @@ arena_dalloc(arena_t *arena, arena_chunk
size = mapelm->npages << pagesize_2pow;
assert((((uintptr_t)ptr) & pagesize_mask) == 0);
- if (opt_junk)
+ if (OPT(junk))
memset(ptr, 0x5a, size);
malloc_mutex_lock(&arena->mtx);
@@ -2666,7 +2535,7 @@ arena_dalloc(arena_t *arena, arena_chunk
malloc_mutex_unlock(&arena->mtx);
}
-static bool
+static void
arena_new(arena_t *arena)
{
unsigned i;
@@ -2735,21 +2604,19 @@ arena_new(arena_t *arena)
#ifdef MALLOC_DEBUG
arena->magic = ARENA_MAGIC;
#endif
-
- return (false);
}
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
-arenas_extend(unsigned ind)
+arenas_extend(void)
{
arena_t *ret;
/* Allocate enough space for trailing bins. */
ret = (arena_t *)base_alloc(sizeof(arena_t)
+ (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
- if (ret != NULL && arena_new(ret) == false) {
- arenas[ind] = ret;
+ if (ret != NULL) {
+ arena_new(ret);
return (ret);
}
/* Only reached if there is an OOM error. */
@@ -2762,7 +2629,7 @@ arenas_extend(unsigned ind)
*/
_malloc_message(getprogname(),
": (malloc) Error initializing arena\n", "", "");
- if (opt_abort)
+ if (OPT(abort))
abort();
return (arenas[0]);
@@ -2814,9 +2681,9 @@ huge_malloc(size_t size)
#endif
malloc_mutex_unlock(&chunks_mtx);
- if (opt_junk)
+ if (OPT(junk))
memset(ret, 0xa5, csize);
- else if (opt_zero)
+ else if (OPT(zero))
memset(ret, 0, csize);
return (ret);
@@ -2894,9 +2761,9 @@ huge_palloc(size_t alignment, size_t siz
#endif
malloc_mutex_unlock(&chunks_mtx);
- if (opt_junk)
+ if (OPT(junk))
memset(ret, 0xa5, chunk_size);
- else if (opt_zero)
+ else if (OPT(zero))
memset(ret, 0, chunk_size);
return (ret);
@@ -2910,10 +2777,10 @@ huge_ralloc(void *ptr, size_t size, size
/* Avoid moving the allocation if the size class would not change. */
if (oldsize > arena_maxclass &&
CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
- if (opt_junk && size < oldsize) {
+ if (OPT(junk) && size < oldsize) {
memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
- size);
- } else if (opt_zero && size > oldsize) {
+ } else if (OPT(zero) && size > oldsize) {
memset((void *)((uintptr_t)ptr + oldsize), 0, size
- oldsize);
}
@@ -2946,7 +2813,7 @@ huge_ralloc(void *ptr, size_t size, size
* tree before we acquire the mutex lock again.
*/
malloc_mutex_lock(&chunks_mtx);
- key.chunk = __DECONST(void *, ptr);
+ key.chunk = __UNCONST(ptr);
node = RB_FIND(chunk_tree_s, &huge, &key);
assert(node != NULL);
assert(node->chunk == ptr);
@@ -2986,10 +2853,10 @@ huge_ralloc(void *ptr, size_t size, size
#endif
malloc_mutex_unlock(&chunks_mtx);
- if (opt_junk && size < oldsize) {
+ if (OPT(junk) && size < oldsize) {
memset((void *)((uintptr_t)newptr + size), 0x5a,
newcsize - size);
- } else if (opt_zero && size > oldsize) {
+ } else if (OPT(zero) && size > oldsize) {
memset((void *)((uintptr_t)newptr + oldsize), 0,
size - oldsize);
}
@@ -3045,7 +2912,7 @@ huge_dalloc(void *ptr)
/* Unmap chunk. */
#ifdef USE_BRK
- if (opt_junk)
+ if (OPT(junk))
memset(node->chunk, 0x5a, node->size);
#endif
chunk_dealloc(node->chunk, node->size);
@@ -3184,7 +3051,7 @@ icalloc(size_t size)
if (ret == NULL)
return (NULL);
- if (opt_junk)
+ if (OPT(junk))
memset(ret, 0, size);
#ifdef USE_BRK
else if ((uintptr_t)ret >= (uintptr_t)brk_base
@@ -3223,7 +3090,7 @@ isalloc(const void *ptr)
malloc_mutex_lock(&chunks_mtx);
/* Extract from tree of huge allocations. */
- key.chunk = __DECONST(void *, ptr);
+ key.chunk = __UNCONST(ptr);
node = RB_FIND(chunk_tree_s, &huge, &key);
assert(node != NULL);
@@ -3273,7 +3140,7 @@ static void
malloc_print_stats(void)
{
- if (opt_print_stats) {
+ if (OPT(print_stats)) {
char s[UMAX2S_BUFSIZE];
_malloc_message("___ Begin malloc statistics ___\n", "", "",
"");
@@ -3288,13 +3155,12 @@ malloc_print_stats(void)
opt_abort ? "A" : "a",
opt_junk ? "J" : "j",
opt_hint ? "H" : "h");
- _malloc_message(opt_utrace ? "PU" : "Pu",
+ _malloc_message(OPT(utrace) ? "PU" : "Pu",
opt_sysv ? "V" : "v",
opt_xmalloc ? "X" : "x",
opt_zero ? "Z\n" : "z\n");
_malloc_message("CPUs: ", size_t2s(ncpus, s), "\n", "");
- _malloc_message("Max arenas: ", size_t2s(narenas, s), "\n", "");
_malloc_message("Pointer size: ", size_t2s(sizeof(void *), s),
"\n", "");
_malloc_message("Quantum size: ", size_t2s(quantum, s), "\n", "");
@@ -3314,7 +3180,7 @@ malloc_print_stats(void)
/* Calculate and print allocated/mapped stats. */
/* arenas. */
- for (i = 0, allocated = 0; i < narenas; i++) {
+ for (i = 0, allocated = 0; i < ncpus; i++) {
if (arenas[i] != NULL) {
malloc_mutex_lock(&arenas[i]->mtx);
allocated +=
@@ -3363,7 +3229,7 @@ malloc_print_stats(void)
huge_allocated);
/* Print stats for each arena. */
- for (i = 0; i < narenas; i++) {
+ for (i = 0; i < ncpus; i++) {
arena = arenas[i];
if (arena != NULL) {
malloc_printf(
@@ -3380,15 +3246,14 @@ malloc_print_stats(void)
}
/*
- * FreeBSD's pthreads implementation calls malloc(3), so the malloc
- * implementation has to take pains to avoid infinite recursion during
- * initialization.
+ * libpthread might call malloc(3), so the malloc implementation has to take
+ * pains to avoid infinite recursion during initialization.
*/
static inline bool
malloc_init(void)
{
- if (malloc_initialized == false)
+ if (__predict_false(malloc_initialized == false))
return (malloc_init_hard());
return (false);
@@ -3432,7 +3297,7 @@ malloc_init_hard(void)
{
long result;
- result = sysconf(_SC_PAGESIZE);
+ result = getpagesize();
assert(result != -1);
pagesize = (unsigned) result;
@@ -3450,7 +3315,7 @@ malloc_init_hard(void)
switch (i) {
case 0:
if ((linklen = readlink("/etc/malloc.conf", buf,
- sizeof(buf) - 1)) != -1) {
+ sizeof(buf) - 1)) != -1) {
/*
* Use the contents of the "/etc/malloc.conf"
* symbolic link's name.
@@ -3479,10 +3344,10 @@ malloc_init_hard(void)
break;
case 2:
if (_malloc_options != NULL) {
- /*
- * Use options that were compiled into the program.
- */
- opts = _malloc_options;
+ /*
+ * Use options that were compiled into the program.
+ */
+ opts = _malloc_options;
} else {
/* No configuration specified. */
buf[0] = '\0';
@@ -3529,12 +3394,6 @@ malloc_init_hard(void)
(int)(sizeof(size_t) << 3))
opt_chunk_2pow++;
break;
- case 'n':
- opt_narenas_lshift--;
- break;
- case 'N':
- opt_narenas_lshift++;
- break;
case 'p':
opt_print_stats = false;
break;
@@ -3596,7 +3455,7 @@ malloc_init_hard(void)
errno = serrno;
/* Take care to call atexit() only once. */
- if (opt_print_stats) {
+ if (OPT(print_stats)) {
/* Print statistics at exit. */
atexit(malloc_print_stats);
}
@@ -3604,13 +3463,13 @@ malloc_init_hard(void)
/* Set variables according to the value of opt_small_max_2pow. */
if (opt_small_max_2pow < opt_quantum_2pow)
opt_small_max_2pow = opt_quantum_2pow;
- small_max = (1 << opt_small_max_2pow);
+ small_max = (1U << opt_small_max_2pow);
/* Set bin-related variables. */
bin_maxclass = (pagesize >> 1);
assert(opt_quantum_2pow >= TINY_MIN_2POW);
ntbins = (unsigned)(opt_quantum_2pow - TINY_MIN_2POW);
- assert(ntbins <= opt_quantum_2pow);
+ assert(ntbins <= (unsigned)opt_quantum_2pow);
nqbins = (unsigned)(small_max >> opt_quantum_2pow);
nsbins = (unsigned)(pagesize_2pow - opt_small_max_2pow - 1);
@@ -3684,37 +3543,8 @@ malloc_init_hard(void)
base_chunk_nodes = NULL;
malloc_mutex_init(&base_mtx);
- if (ncpus > 1) {
- /*
- * For SMP systems, create four times as many arenas as there
- * are CPUs by default.
- */
- opt_narenas_lshift += 2;
- }
-
- /* Determine how many arenas to use. */
- narenas = ncpus;
- if (opt_narenas_lshift > 0) {
- if ((narenas << opt_narenas_lshift) > narenas)
- narenas <<= opt_narenas_lshift;
- /*
- * Make sure not to exceed the limits of what base_malloc()
- * can handle.
- */
- if (narenas * sizeof(arena_t *) > chunksize)
- narenas = (unsigned)(chunksize / sizeof(arena_t *));
- } else if (opt_narenas_lshift < 0) {
- if ((narenas << opt_narenas_lshift) < narenas)
- narenas <<= opt_narenas_lshift;
- /* Make sure there is at least one arena. */
- if (narenas == 0)
- narenas = 1;
- }
-
- next_arena = 0;
-
/* Allocate and initialize arenas. */
- arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
+ arenas = (arena_t **)base_alloc(sizeof(arena_t *) * ncpus);
if (arenas == NULL) {
malloc_mutex_unlock(&init_lock);
return (true);
@@ -3723,14 +3553,13 @@ malloc_init_hard(void)
* Zero the array. In practice, this should always be pre-zeroed,
* since it was just mmap()ed, but let's be sure.
*/
- memset(arenas, 0, sizeof(arena_t *) * narenas);
+ memset(arenas, 0, sizeof(arena_t *) * ncpus);
/*
* Initialize one arena here. The rest are lazily created in
* arena_choose_hard().
*/
- arenas_extend(0);
- if (arenas[0] == NULL) {
+ if ((arenas[0] = arenas_extend()) == NULL) {
malloc_mutex_unlock(&init_lock);
return (true);
}
@@ -3755,13 +3584,13 @@ malloc(size_t size)
{
void *ret;
- if (malloc_init()) {
+ if (__predict_false(malloc_init())) {
ret = NULL;
goto RETURN;
}
- if (size == 0) {
- if (opt_sysv == false)
+ if (__predict_false(size == 0)) {
+ if (NOT_OPT(sysv))
size = 1;
else {
ret = NULL;
@@ -3772,8 +3601,8 @@ malloc(size_t size)
ret = imalloc(size);
RETURN:
- if (ret == NULL) {
- if (opt_xmalloc) {
+ if (__predict_false(ret == NULL)) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in malloc(): out of memory\n", "",
"");
@@ -3792,13 +3621,13 @@ posix_memalign(void **memptr, size_t ali
int ret;
void *result;
- if (malloc_init())
+ if (__predict_false(malloc_init()))
result = NULL;
else {
/* Make sure that alignment is a large enough power of 2. */
if (((alignment - 1) & alignment) != 0
|| alignment < sizeof(void *)) {
- if (opt_xmalloc) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in posix_memalign(): "
"invalid alignment\n", "", "");
@@ -3812,8 +3641,8 @@ posix_memalign(void **memptr, size_t ali
result = ipalloc(alignment, size);
}
- if (result == NULL) {
- if (opt_xmalloc) {
+ if (__predict_false(result == NULL)) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in posix_memalign(): out of memory\n",
"", "");
@@ -3837,15 +3666,15 @@ calloc(size_t num, size_t size)
void *ret;
size_t num_size;
- if (malloc_init()) {
+ if (__predict_false(malloc_init())) {
num_size = 0;
ret = NULL;
goto RETURN;
}
num_size = num * size;
- if (num_size == 0) {
- if ((opt_sysv == false) && ((num == 0) || (size == 0)))
+ if (__predict_false(num_size == 0)) {
+ if (NOT_OPT(sysv) && ((num == 0) || (size == 0)))
num_size = 1;
else {
ret = NULL;
@@ -3868,7 +3697,7 @@ calloc(size_t num, size_t size)
RETURN:
if (ret == NULL) {
- if (opt_xmalloc) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in calloc(): out of memory\n", "",
"");
@@ -3886,8 +3715,8 @@ realloc(void *ptr, size_t size)
{
void *ret;
- if (size == 0) {
- if (opt_sysv == false)
+ if (__predict_false(size == 0)) {
+ if (NOT_OPT(sysv))
size = 1;
else {
if (ptr != NULL)
@@ -3897,13 +3726,13 @@ realloc(void *ptr, size_t size)
}
}
- if (ptr != NULL) {
+ if (__predict_true(ptr != NULL)) {
assert(malloc_initialized);
ret = iralloc(ptr, size);
if (ret == NULL) {
- if (opt_xmalloc) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in realloc(): out of "
"memory\n", "", "");
@@ -3912,13 +3741,13 @@ realloc(void *ptr, size_t size)
errno = ENOMEM;
}
} else {
- if (malloc_init())
+ if (__predict_false(malloc_init()))
ret = NULL;
else
ret = imalloc(size);
if (ret == NULL) {
- if (opt_xmalloc) {
+ if (OPT(xmalloc)) {
_malloc_message(getprogname(),
": (malloc) Error in realloc(): out of "
"memory\n", "", "");
@@ -3982,7 +3811,7 @@ _malloc_prefork(void)
/* Acquire all mutexes in a safe order. */
malloc_mutex_lock(&init_lock);
malloc_mutex_lock(&arenas_mtx);
- for (i = 0; i < narenas; i++) {
+ for (i = 0; i < ncpus; i++) {
if (arenas[i] != NULL)
malloc_mutex_lock(&arenas[i]->mtx);
}
@@ -4005,7 +3834,7 @@ _malloc_postfork(void)
malloc_mutex_unlock(&base_mtx);
malloc_mutex_unlock(&chunks_mtx);
- for (i = narenas; i-- > 0; ) {
+ for (i = ncpus; i-- > 0; ) {
if (arenas[i] != NULL)
malloc_mutex_unlock(&arenas[i]->mtx);
}
@@ -4025,7 +3854,7 @@ _malloc_postfork_child(void)
malloc_mutex_init(&base_mtx);
malloc_mutex_init(&chunks_mtx);
- for (i = narenas; i-- > 0; ) {
+ for (i = ncpus; i-- > 0; ) {
if (arenas[i] != NULL)
malloc_mutex_init(&arenas[i]->mtx);
}
