It is advantageous to have all defragmentable slabs together at the
beginning of the list of slabs so that there is no need to scan the
complete list. Put defragmentable caches first when adding a slab cache
and others last.
Determine the maximum number of objects in defragmentable slabs. This allows
the sizing of the array holding refs to objects in a slab later.
Signed-off-by: Christoph Lameter <c...@linux.com>
---
mm/slub.c | 26 ++++++++++++++++++++++++--
1 file changed, 24 insertions(+), 2 deletions(-)
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c
+++ linux/mm/slub.c
@@ -196,6 +196,9 @@ static inline bool kmem_cache_has_cpu_pa
/* Use cmpxchg_double */
#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
+/* Maximum objects in defragmentable slabs */
+static unsigned int max_defrag_slab_objects;
+
/*
* Tracking user of a slab.
*/
@@ -4310,22 +4313,45 @@ int __kmem_cache_create(struct kmem_cach
return err;
}
+/*
+ * Allocate a slab scratch space that is sufficient to keep at least
+ * max_defrag_slab_objects pointers to individual objects and also a bitmap
+ * for max_defrag_slab_objects.
+ */
+static inline void *alloc_scratch(void)
+{
+ return kmalloc(max_defrag_slab_objects * sizeof(void *) +
+ BITS_TO_LONGS(max_defrag_slab_objects) * sizeof(unsigned long),
+ GFP_KERNEL);
+}
+
void kmem_cache_setup_mobility(struct kmem_cache *s,
kmem_isolate_func isolate, kmem_migrate_func migrate)
{
+ int max_objects = oo_objects(s->max);
+
/*
* Defragmentable slabs must have a ctor otherwise objects may be
* in an undetermined state after they are allocated.
*/
BUG_ON(!s->ctor);
+
+ mutex_lock(&slab_mutex);
+
s->isolate = isolate;
s->migrate = migrate;
+
/*
* Sadly serialization requirements currently mean that we have
* to disable fast cmpxchg based processing.
*/
s->flags &= ~__CMPXCHG_DOUBLE;
+ list_move(&s->list, &slab_caches); /* Move to top */
+ if (max_objects > max_defrag_slab_objects)
+ max_defrag_slab_objects = max_objects;
+
+ mutex_unlock(&slab_mutex);
}
EXPORT_SYMBOL(kmem_cache_setup_mobility);
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c
+++ linux/mm/slab_common.c
@@ -393,7 +393,7 @@ static struct kmem_cache *create_cache(c
goto out_free_cache;
s->refcount = 1;
- list_add(&s->list, &slab_caches);
+ list_add_tail(&s->list, &slab_caches);
memcg_link_cache(s);
out:
if (err)
