We have now in place a mechanism for adding callbacks to a cache in
order to be able to implement object migration.
Add a function __move() that implements SMO by moving all objects in a
slab page using the isolate/migrate callback methods.
Co-developed-by: Christoph Lameter <c...@linux.com>
Signed-off-by: Tobin C. Harding <to...@kernel.org>
---
mm/slub.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 85 insertions(+)
diff --git a/mm/slub.c b/mm/slub.c
index 0133168d1089..6ce866b420f1 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4325,6 +4325,91 @@ int __kmem_cache_create(struct kmem_cache *s,
slab_flags_t flags)
return err;
}
+/*
+ * Allocate a slab scratch space that is sufficient to keep pointers to
+ * individual objects for all objects in cache and also a bitmap for the
+ * objects (used to mark which objects are active).
+ */
+static inline void *alloc_scratch(struct kmem_cache *s)
+{
+ unsigned int size = oo_objects(s->max);
+
+ return kmalloc(size * sizeof(void *) +
+ BITS_TO_LONGS(size) * sizeof(unsigned long),
+ GFP_KERNEL);
+}
+
+/*
+ * __move() - Move all objects in the given slab.
+ * @page: The slab we are working on.
+ * @scratch: Pointer to scratch space.
+ * @node: The target node to move objects to.
+ *
+ * If the target node is not the current node then the object is moved
+ * to the target node. If the target node is the current node then this
+ * is an effective way of defragmentation since the current slab page
+ * with its object is exempt from allocation.
+ */
+static void __move(struct page *page, void *scratch, int node)
+{
+ unsigned long objects;
+ struct kmem_cache *s;
+ unsigned long flags;
+ unsigned long *map;
+ void *private;
+ int count;
+ void *p;
+ void **vector = scratch;
+ void *addr = page_address(page);
+
+ local_irq_save(flags);
+ slab_lock(page);
+
+ BUG_ON(!PageSlab(page)); /* Must be s slab page */
+ BUG_ON(!page->frozen); /* Slab must have been frozen earlier */
+
+ s = page->slab_cache;
+ objects = page->objects;
+ map = scratch + objects * sizeof(void **);
+
+ /* Determine used objects */
+ bitmap_fill(map, objects);
+ for (p = page->freelist; p; p = get_freepointer(s, p))
+ __clear_bit(slab_index(p, s, addr), map);
+
+ /* Build vector of pointers to objects */
+ count = 0;
+ memset(vector, 0, objects * sizeof(void **));
+ for_each_object(p, s, addr, objects)
+ if (test_bit(slab_index(p, s, addr), map))
+ vector[count++] = p;
+
+ if (s->isolate)
+ private = s->isolate(s, vector, count);
+ else
+ /* Objects do not need to be isolated */
+ private = NULL;
+
+ /*
+ * Pinned the objects. Now we can drop the slab lock. The slab
+ * is frozen so it cannot vanish from under us nor will
+ * allocations be performed on the slab. However, unlocking the
+ * slab will allow concurrent slab_frees to proceed. So the
+ * subsystem must have a way to tell from the content of the
+ * object that it was freed.
+ *
+ * If neither RCU nor ctor is being used then the object may be
+ * modified by the allocator after being freed which may disrupt
+ * the ability of the migrate function to tell if the object is
+ * free or not.
+ */
+ slab_unlock(page);
+ local_irq_restore(flags);
+
+ /* Perform callback to move the objects */
+ s->migrate(s, vector, count, node, private);
+}
+
void kmem_cache_setup_mobility(struct kmem_cache *s,
kmem_cache_isolate_func isolate,
kmem_cache_migrate_func migrate)
--
2.21.0
