Currently, when slabs are marked with debug options, the allocation path will skip using CPU slabs. This has a definite performance impact. Add an option to allow debugging to happen on the fast path.
Signed-off-by: Laura Abbott <labb...@fedoraproject.org> --- init/Kconfig | 12 +++++ mm/slub.c | 164 +++++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 161 insertions(+), 15 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index 2232080..6d807e7 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1674,6 +1674,18 @@ config SLUB_DEBUG SLUB sysfs support. /sys/slab will not exist and there will be no support for cache validation etc. +config SLUB_DEBUG_FASTPATH + bool "Allow SLUB debugging to utilize the fastpath" + depends on SLUB_DEBUG + help + SLUB_DEBUG forces all allocations to utilize the slow path which + is a performance penalty. Turning on this option lets the debugging + use the fast path. This helps the performance when debugging + features are turned on. If you aren't planning on utilizing any + of the SLUB_DEBUG features, you should say N here. + + If unsure, say N + config COMPAT_BRK bool "Disable heap randomization" default y diff --git a/mm/slub.c b/mm/slub.c index 6ddba32..a47e615 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -898,9 +898,10 @@ static int check_slab(struct kmem_cache *s, struct page *page) * Determine if a certain object on a page is on the freelist. Must hold the * slab lock to guarantee that the chains are in a consistent state. */ -static int on_freelist(struct kmem_cache *s, struct page *page, void *search) +static int on_freelist(struct kmem_cache *s, struct page *page, void *search, + void *cpu_freelist) { - int nr = 0; + int nr = 0, cpu_nr = 0; void *fp; void *object = NULL; int max_objects; @@ -928,6 +929,29 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search) nr++; } + fp = cpu_freelist; + while (fp && cpu_nr <= page->objects) { + if (fp == search) + return 1; + if (!check_valid_pointer(s, page, fp)) { + if (object) { + object_err(s, page, object, + "Freechain corrupt"); + set_freepointer(s, object, NULL); + } else { + slab_err(s, page, "Freepointer corrupt"); + page->freelist = NULL; + page->inuse = page->objects; + slab_fix(s, "Freelist cleared"); + return 0; + } + break; + } + object = fp; + fp = get_freepointer(s, object); + cpu_nr++; + } + max_objects = order_objects(compound_order(page), s->size, s->reserved); if (max_objects > MAX_OBJS_PER_PAGE) max_objects = MAX_OBJS_PER_PAGE; @@ -1034,6 +1058,7 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page, init_tracking(s, object); } +/* Must be not be called when migration can happen */ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *page, void *object, unsigned long addr) @@ -1070,10 +1095,51 @@ bad: return 0; } +#ifdef SLUB_DEBUG_FASTPATH +static noinline int alloc_debug_processing_fastpath(struct kmem_cache *s, + struct kmem_cache_cpu *c, + struct page *page, + void *object, unsigned long tid, + unsigned long addr) +{ + unsigned long flags; + int ret = 0; + + preempt_disable(); + local_irq_save(flags); + + /* + * We've now disabled preemption and IRQs but we still need + * to check that this is the right CPU + */ + if (!this_cpu_cmpxchg_double(s->cpu_slab->freelist, s->cpu_slab->tid, + c->freelist, tid, + c->freelist, tid)) + goto out; + + ret = alloc_debug_processing(s, page, object, addr); + +out: + local_irq_restore(flags); + preempt_enable(); + return ret; +} +#else +static noinline int alloc_debug_processing_fastpath(struct kmem_cache *s, + struct kmem_cache_cpu *c, + struct page *page, + void *object, unsigned long tid, + unsigned long addr) +{ + return 1; +} +#endif + /* Supports checking bulk free of a constructed freelist */ static noinline int free_debug_processing( struct kmem_cache *s, struct page *page, void *head, void *tail, int bulk_cnt, + void *cpu_freelist, unsigned long addr) { struct kmem_cache_node *n = get_node(s, page_to_nid(page)); @@ -1095,7 +1161,7 @@ next_object: goto fail; } - if (on_freelist(s, page, object)) { + if (on_freelist(s, page, object, cpu_freelist)) { object_err(s, page, object, "Object already free"); goto fail; } @@ -1144,6 +1210,53 @@ fail: return 0; } +#ifdef CONFIG_SLUB_DEBUG_FASTPATH +static noinline int free_debug_processing_fastpath( + struct kmem_cache *s, + struct kmem_cache_cpu *c, + struct page *page, + void *head, void *tail, int bulk_cnt, + unsigned long tid, + unsigned long addr) +{ + int ret = 0; + unsigned long flags; + + preempt_disable(); + local_irq_save(flags); + + /* + * We've now disabled preemption and IRQs but we still need + * to check that this is the right CPU + */ + if (!this_cpu_cmpxchg_double(s->cpu_slab->freelist, s->cpu_slab->tid, + c->freelist, tid, + c->freelist, tid)) + goto out; + + + ret = free_debug_processing(s, page, head, tail, bulk_cnt, + c->freelist, addr); + +out: + local_irq_restore(flags); + preempt_enable(); + return ret; +} +#else +static inline int free_debug_processing_fastpath( + struct kmem_cache *s, + struct kmem_cache_cpu *c, + struct page *page, + void *head, void *tail, int bulk_cnt, + unsigned long tid, + unsigned long addr) +{ + return 1; +} +#endif + + static int __init setup_slub_debug(char *str) { slub_debug = DEBUG_DEFAULT_FLAGS; @@ -1234,8 +1347,8 @@ static inline int alloc_debug_processing(struct kmem_cache *s, static inline int free_debug_processing( struct kmem_cache *s, struct page *page, - void *head, void *tail, int bulk_cnt, - unsigned long addr, unsigned long *flags) { return NULL; } + void *head, void *tail, int bulk_cnt, void *cpu_freelist, + unsigned long addr, unsigned long *flags) { return 0; } static inline int slab_pad_check(struct kmem_cache *s, struct page *page) { return 1; } @@ -2352,7 +2465,8 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) * already disabled (which is the case for bulk allocation). */ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) + unsigned long addr, struct kmem_cache_cpu *c, + bool debug_fail) { void *freelist; struct page *page; @@ -2382,7 +2496,7 @@ redo: * PFMEMALLOC but right now, we are losing the pfmemalloc * information when the page leaves the per-cpu allocator */ - if (unlikely(!pfmemalloc_match(page, gfpflags))) { + if (unlikely(debug_fail || !pfmemalloc_match(page, gfpflags))) { deactivate_slab(s, page, c->freelist); c->page = NULL; c->freelist = NULL; @@ -2433,7 +2547,9 @@ new_slab: } page = c->page; - if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) + + if (!IS_ENABLED(CONFIG_SLUB_DEBUG_FASTPATH) && + likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) goto load_freelist; /* Only entered in the debug case */ @@ -2441,6 +2557,10 @@ new_slab: !alloc_debug_processing(s, page, freelist, addr)) goto new_slab; /* Slab failed checks. Next slab needed */ + if (IS_ENABLED(CONFIG_SLUB_DEBUG_FASTPATH) && + likely(pfmemalloc_match(page, gfpflags))) + goto load_freelist; + deactivate_slab(s, page, get_freepointer(s, freelist)); c->page = NULL; c->freelist = NULL; @@ -2452,7 +2572,8 @@ new_slab: * cpu changes by refetching the per cpu area pointer. */ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) + unsigned long addr, struct kmem_cache_cpu *c, + bool debug_fail) { void *p; unsigned long flags; @@ -2467,7 +2588,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, c = this_cpu_ptr(s->cpu_slab); #endif - p = ___slab_alloc(s, gfpflags, node, addr, c); + p = ___slab_alloc(s, gfpflags, node, addr, c, debug_fail); local_irq_restore(flags); return p; } @@ -2489,6 +2610,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s, struct kmem_cache_cpu *c; struct page *page; unsigned long tid; + bool debug_fail = false; s = slab_pre_alloc_hook(s, gfpflags); if (!s) @@ -2529,12 +2651,18 @@ redo: object = c->freelist; page = c->page; - if (unlikely(!object || !node_match(page, node))) { - object = __slab_alloc(s, gfpflags, node, addr, c); + if (unlikely(debug_fail || !object || !node_match(page, node))) { + object = __slab_alloc(s, gfpflags, node, addr, c, debug_fail); stat(s, ALLOC_SLOWPATH); } else { void *next_object = get_freepointer_safe(s, object); + + if (kmem_cache_debug(s) && !alloc_debug_processing_fastpath(s, c, page, object, tid, addr)) { + debug_fail = true; + goto redo; + } + /* * The cmpxchg will only match if there was no additional * operation and if we are on the right processor. @@ -2557,6 +2685,7 @@ redo: note_cmpxchg_failure("slab_alloc", s, tid); goto redo; } + prefetch_freepointer(s, next_object); stat(s, ALLOC_FASTPATH); } @@ -2649,9 +2778,10 @@ static void __slab_free(struct kmem_cache *s, struct page *page, stat(s, FREE_SLOWPATH); if (kmem_cache_debug(s) && - !free_debug_processing(s, page, head, tail, cnt, addr)) + !free_debug_processing(s, page, head, tail, cnt, NULL, addr)) return; + do { if (unlikely(n)) { spin_unlock_irqrestore(&n->list_lock, flags); @@ -2790,6 +2920,10 @@ redo: barrier(); if (likely(page == c->page)) { + if (kmem_cache_debug(s) && + !free_debug_processing_fastpath(s, c, page, head, tail_obj, cnt, tid, addr)) + return; + set_freepointer(s, tail_obj, c->freelist); if (unlikely(!this_cpu_cmpxchg_double( @@ -2938,7 +3072,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, * of re-populating per CPU c->freelist */ p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, - _RET_IP_, c); + _RET_IP_, c, false); if (unlikely(!p[i])) goto error; @@ -4094,7 +4228,7 @@ static int validate_slab(struct kmem_cache *s, struct page *page, void *addr = page_address(page); if (!check_slab(s, page) || - !on_freelist(s, page, NULL)) + !on_freelist(s, page, NULL, NULL)) return 0; /* Now we know that a valid freelist exists */ -- 2.5.0