[RFC PATCH v1 03/11] mm: Account the top tier memory usage per cgroup

[Tim Chen]

For each memory cgroup, account its usage of the
top tier memory at the time a top tier page is assigned and
uncharged from the cgroup.

Signed-off-by: Tim Chen <tim.c.c...@linux.intel.com>
---
 include/linux/memcontrol.h |  1 +
 mm/memcontrol.c            | 39 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 25d8b9acec7c..609d8590950c 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -225,6 +225,7 @@ struct mem_cgroup {
        /* Legacy consumer-oriented counters */
        struct page_counter kmem;               /* v1 only */
        struct page_counter tcpmem;             /* v1 only */
+       struct page_counter toptier;
 
        /* Range enforcement for interrupt charges */
        struct work_struct high_work;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 9a9d677a6654..fe7bb8613f5a 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -253,6 +253,13 @@ struct cgroup_subsys_state *vmpressure_to_css(struct 
vmpressure *vmpr)
        return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
 }
 
+static inline bool top_tier(struct page *page)
+{
+       int nid = page_to_nid(page);
+
+       return node_state(nid, N_TOPTIER);
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 extern spinlock_t css_set_lock;
 
@@ -951,6 +958,23 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup 
*memcg,
        __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
 }
 
+static inline void mem_cgroup_charge_toptier(struct mem_cgroup *memcg,
+                                        struct page *page,
+                                        int nr_pages)
+{
+       if (!top_tier(page))
+               return;
+
+       if (nr_pages >= 0)
+               page_counter_charge(&memcg->toptier,
+                                  (unsigned long) nr_pages);
+       else {
+               nr_pages = -nr_pages;
+               page_counter_uncharge(&memcg->toptier,
+                                  (unsigned long) nr_pages);
+       }
+}
+
 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
                                       enum mem_cgroup_events_target target)
 {
@@ -2932,6 +2956,7 @@ static void commit_charge(struct page *page, struct 
mem_cgroup *memcg)
         * - exclusive reference
         */
        page->memcg_data = (unsigned long)memcg;
+       mem_cgroup_charge_toptier(memcg, page, thp_nr_pages(page));
 }
 
 #ifdef CONFIG_MEMCG_KMEM
@@ -3138,6 +3163,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t 
gfp, int order)
                if (!ret) {
                        page->memcg_data = (unsigned long)memcg |
                                MEMCG_DATA_KMEM;
+                       mem_cgroup_charge_toptier(memcg, page, 1 << order);
                        return 0;
                }
                css_put(&memcg->css);
@@ -3161,6 +3187,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int 
order)
        VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
        __memcg_kmem_uncharge(memcg, nr_pages);
        page->memcg_data = 0;
+       mem_cgroup_charge_toptier(memcg, page, -nr_pages);
        css_put(&memcg->css);
 }
 
@@ -5389,11 +5416,13 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state 
*parent_css)
 
                page_counter_init(&memcg->memory, &parent->memory);
                page_counter_init(&memcg->swap, &parent->swap);
+               page_counter_init(&memcg->toptier, &parent->toptier);
                page_counter_init(&memcg->kmem, &parent->kmem);
                page_counter_init(&memcg->tcpmem, &parent->tcpmem);
        } else {
                page_counter_init(&memcg->memory, NULL);
                page_counter_init(&memcg->swap, NULL);
+               page_counter_init(&memcg->toptier, NULL);
                page_counter_init(&memcg->kmem, NULL);
                page_counter_init(&memcg->tcpmem, NULL);
 
@@ -5745,6 +5774,8 @@ static int mem_cgroup_move_account(struct page *page,
        css_put(&from->css);
 
        page->memcg_data = (unsigned long)to;
+       mem_cgroup_charge_toptier(to, page, nr_pages);
+       mem_cgroup_charge_toptier(from, page, -nr_pages);
 
        __unlock_page_memcg(from);
 
@@ -6832,6 +6863,7 @@ struct uncharge_gather {
        unsigned long nr_pages;
        unsigned long pgpgout;
        unsigned long nr_kmem;
+       unsigned long nr_toptier;
        struct page *dummy_page;
 };
 
@@ -6846,6 +6878,7 @@ static void uncharge_batch(const struct uncharge_gather 
*ug)
 
        if (!mem_cgroup_is_root(ug->memcg)) {
                page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
+               page_counter_uncharge(&ug->memcg->toptier, ug->nr_toptier);
                if (do_memsw_account())
                        page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
                if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
@@ -6891,6 +6924,8 @@ static void uncharge_page(struct page *page, struct 
uncharge_gather *ug)
 
        nr_pages = compound_nr(page);
        ug->nr_pages += nr_pages;
+       if (top_tier(page))
+               ug->nr_toptier += nr_pages;
 
        if (PageMemcgKmem(page))
                ug->nr_kmem += nr_pages;
@@ -7216,8 +7251,10 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t 
entry)
 
        page->memcg_data = 0;
 
-       if (!mem_cgroup_is_root(memcg))
+       if (!mem_cgroup_is_root(memcg)) {
                page_counter_uncharge(&memcg->memory, nr_entries);
+               mem_cgroup_charge_toptier(memcg, page, -nr_entries);
+       }
 
        if (!cgroup_memory_noswap && memcg != swap_memcg) {
                if (!mem_cgroup_is_root(swap_memcg))
-- 
2.20.1