[PATCH 02/10] mm: Maintain memcg-aware shrinkers in mcg_shrinkers array

[Kirill Tkhai]

The patch introduces mcg_shrinkers array to keep memcg-aware
shrinkers in order of their shrinker::id.

This allows to access the shrinkers dirrectly by the id,
without iteration over shrinker_list list.

Signed-off-by: Kirill Tkhai <ktk...@virtuozzo.com>
---
 mm/vmscan.c |   89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 81 insertions(+), 8 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 91b5120b924f..97ce4f342fab 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -163,6 +163,46 @@ static DECLARE_RWSEM(shrinker_rwsem);
 static DEFINE_IDA(bitmap_id_ida);
 static DECLARE_RWSEM(bitmap_rwsem);
 static int bitmap_id_start;
+static int bitmap_nr_ids;
+static struct shrinker **mcg_shrinkers;
+
+static int expand_shrinkers_array(int old_nr, int nr)
+{
+       struct shrinker **new;
+       int old_size, size;
+
+       size = nr * sizeof(struct shrinker *);
+       new = kvmalloc(size, GFP_KERNEL);
+       if (!new)
+               return -ENOMEM;
+
+       old_size = old_nr * sizeof(struct shrinker *);
+       memset((void *)new + old_size, 0, size - old_size);
+
+       down_write(&shrinker_rwsem);
+       memcpy((void *)new, mcg_shrinkers, old_size);
+       swap(new, mcg_shrinkers);
+       up_write(&shrinker_rwsem);
+
+       kvfree(new);
+       return 0;
+}
+
+static int expand_shrinker_id(int id)
+{
+       if (likely(id < bitmap_nr_ids))
+               return 0;
+
+       id = bitmap_nr_ids * 2;
+       if (id == 0)
+               id = BITS_PER_BYTE;
+
+       if (expand_shrinkers_array(bitmap_nr_ids, id))
+               return -ENOMEM;
+
+       bitmap_nr_ids = id;
+       return 0;
+}
 
 static int alloc_shrinker_id(struct shrinker *shrinker)
 {
@@ -175,8 +215,13 @@ static int alloc_shrinker_id(struct shrinker *shrinker)
        down_write(&bitmap_rwsem);
        ret = ida_get_new_above(&bitmap_id_ida, bitmap_id_start, &id);
        if (!ret) {
-               shrinker->id = id;
-               bitmap_id_start = shrinker->id + 1;
+               if (expand_shrinker_id(id)) {
+                       ida_remove(&bitmap_id_ida, id);
+                       ret = -ENOMEM;
+               } else {
+                       shrinker->id = id;
+                       bitmap_id_start = shrinker->id + 1;
+               }
        }
        up_write(&bitmap_rwsem);
        if (ret == -EAGAIN)
@@ -198,6 +243,24 @@ static void free_shrinker_id(struct shrinker *shrinker)
                bitmap_id_start = id;
        up_write(&bitmap_rwsem);
 }
+
+static void add_shrinker(struct shrinker *shrinker)
+{
+       down_write(&shrinker_rwsem);
+       if (shrinker->flags & SHRINKER_MEMCG_AWARE)
+               mcg_shrinkers[shrinker->id] = shrinker;
+       list_add_tail(&shrinker->list, &shrinker_list);
+       up_write(&shrinker_rwsem);
+}
+
+static void del_shrinker(struct shrinker *shrinker)
+{
+       down_write(&shrinker_rwsem);
+       if (shrinker->flags & SHRINKER_MEMCG_AWARE)
+               mcg_shrinkers[shrinker->id] = NULL;
+       list_del(&shrinker->list);
+       up_write(&shrinker_rwsem);
+}
 #else /* CONFIG_MEMCG && !CONFIG_SLOB */
 static int alloc_shrinker_id(struct shrinker *shrinker)
 {
@@ -207,6 +270,20 @@ static int alloc_shrinker_id(struct shrinker *shrinker)
 static void free_shrinker_id(struct shrinker *shrinker)
 {
 }
+
+static void add_shrinker(struct shrinker *shrinker)
+{
+       down_write(&shrinker_rwsem);
+       list_add_tail(&shrinker->list, &shrinker_list);
+       up_write(&shrinker_rwsem);
+}
+
+static void del_shrinker(struct shrinker *shrinker)
+{
+       down_write(&shrinker_rwsem);
+       list_del(&shrinker->list);
+       up_write(&shrinker_rwsem);
+}
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
 
 #ifdef CONFIG_MEMCG
@@ -322,9 +399,7 @@ int register_shrinker(struct shrinker *shrinker)
        if (alloc_shrinker_id(shrinker))
                goto free_deferred;
 
-       down_write(&shrinker_rwsem);
-       list_add_tail(&shrinker->list, &shrinker_list);
-       up_write(&shrinker_rwsem);
+       add_shrinker(shrinker);
        return 0;
 
 free_deferred:
@@ -341,9 +416,7 @@ void unregister_shrinker(struct shrinker *shrinker)
 {
        if (!shrinker->nr_deferred)
                return;
-       down_write(&shrinker_rwsem);
-       list_del(&shrinker->list);
-       up_write(&shrinker_rwsem);
+       del_shrinker(shrinker);
        free_shrinker_id(shrinker);
        kfree(shrinker->nr_deferred);
        shrinker->nr_deferred = NULL;