Teach kfree_rcu_sheaf() how to handle the !allow_spin case. Similar to __pcs_replace_full_main(), try to get an empty sheaf from pcs->spare or the barn, but don't add !allow_spin support for alloc_empty_sheaf() and fail early instead.
Since call_rcu() does not support NMI contexts, kfree_rcu_sheaf() fails when the rcu sheaf becomes full. Signed-off-by: Harry Yoo <[email protected]> --- mm/slab.h | 2 +- mm/slab_common.c | 7 +++---- mm/slub.c | 14 ++++++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index 71c7261bf822..5e05a684258f 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -404,7 +404,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT)); } -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, bool allow_spin); void flush_all_rcu_sheaves(void); void flush_rcu_sheaves_on_cache(struct kmem_cache *s); diff --git a/mm/slab_common.c b/mm/slab_common.c index 9d7801e5cb73..3ee3cf8da304 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1675,7 +1675,7 @@ static void kfree_rcu_work(struct work_struct *work) kvfree_rcu_list(head); } -static bool kfree_rcu_sheaf(void *obj) +static bool kfree_rcu_sheaf(void *obj, bool allow_spin) { struct kmem_cache *s; struct slab *slab; @@ -1689,7 +1689,7 @@ static bool kfree_rcu_sheaf(void *obj) s = slab->slab_cache; if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) - return __kfree_rcu_sheaf(s, obj); + return __kfree_rcu_sheaf(s, obj, allow_spin); return false; } @@ -2044,8 +2044,7 @@ void kvfree_call_rcu_ptr(struct rcu_ptr *head, void *ptr, bool allow_spin) IS_ENABLED(CONFIG_DEBUG_KMEMLEAK))) goto defer_free; - if (!IS_ENABLED(CONFIG_PREEMPT_RT) && - (allow_spin && kfree_rcu_sheaf(ptr))) + if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr, allow_spin)) return; // Queue the object but don't yet schedule the batch. diff --git a/mm/slub.c b/mm/slub.c index ac7bc7e1163f..48f5d6dd3767 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5783,7 +5783,7 @@ static void rcu_free_sheaf(struct rcu_head *head) */ static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); -bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) +bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, bool allow_spin) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; @@ -5821,7 +5821,7 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) goto fail; } - empty = barn_get_empty_sheaf(barn, true); + empty = barn_get_empty_sheaf(barn, allow_spin); if (empty) { pcs->rcu_free = empty; @@ -5830,6 +5830,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); + /* It's easier to fall back than trying harder with !allow_spin */ + if (!allow_spin) + goto fail; + empty = alloc_empty_sheaf(s, GFP_NOWAIT); if (!empty) @@ -5861,6 +5865,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) if (likely(rcu_sheaf->size < s->sheaf_capacity)) { rcu_sheaf = NULL; } else { + if (unlikely(!allow_spin)) { + /* call_rcu() does not support NMI context */ + rcu_sheaf->size--; + local_unlock(&s->cpu_sheaves->lock); + goto fail; + } pcs->rcu_free = NULL; rcu_sheaf->node = numa_mem_id(); } -- 2.43.0
