Currently we use per-cpu stocks to do precharges of the ->memory and ->memsw counters. Do this for the ->kmem and ->cache as well to decrease contention on these counters as well.
https://jira.sw.ru/browse/PSBM-101300 Signed-off-by: Andrey Ryabinin <aryabi...@virtuozzo.com> --- mm/memcontrol.c | 75 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 134cb27307f2..b3f97309ca39 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2023,6 +2023,8 @@ EXPORT_SYMBOL(unlock_page_memcg); struct memcg_stock_pcp { struct mem_cgroup *cached; /* this never be root cgroup */ unsigned int nr_pages; + unsigned int cache_nr_pages; + unsigned int kmem_nr_pages; struct work_struct work; unsigned long flags; #define FLUSHING_CACHED_CHARGE 0 @@ -2041,7 +2043,8 @@ static DEFINE_MUTEX(percpu_charge_mutex); * * returns true if successful, false otherwise. */ -static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) +static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages, + bool cache, bool kmem) { struct memcg_stock_pcp *stock; unsigned long flags; @@ -2053,9 +2056,19 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) local_irq_save(flags); stock = this_cpu_ptr(&memcg_stock); - if (memcg == stock->cached && stock->nr_pages >= nr_pages) { - stock->nr_pages -= nr_pages; - ret = true; + if (memcg == stock->cached) { + if (cache && stock->cache_nr_pages >= nr_pages) { + stock->cache_nr_pages -= nr_pages; + ret = true; + } + if (kmem && stock->kmem_nr_pages >= nr_pages) { + stock->kmem_nr_pages -= nr_pages; + ret = true; + } + if (!cache && !kmem && stock->nr_pages >= nr_pages) { + stock->nr_pages -= nr_pages; + ret = true; + } } local_irq_restore(flags); @@ -2069,13 +2082,21 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) static void drain_stock(struct memcg_stock_pcp *stock) { struct mem_cgroup *old = stock->cached; + unsigned long nr_pages = stock->nr_pages + stock->cache_nr_pages + stock->kmem_nr_pages; + + if (stock->cache_nr_pages) + page_counter_uncharge(&old->cache, stock->cache_nr_pages); + if (stock->kmem_nr_pages) + page_counter_uncharge(&old->kmem, stock->kmem_nr_pages); - if (stock->nr_pages) { - page_counter_uncharge(&old->memory, stock->nr_pages); + if (nr_pages) { + page_counter_uncharge(&old->memory, nr_pages); if (do_memsw_account()) - page_counter_uncharge(&old->memsw, stock->nr_pages); + page_counter_uncharge(&old->memsw, nr_pages); css_put_many(&old->css, stock->nr_pages); stock->nr_pages = 0; + stock->kmem_nr_pages = 0; + stock->cache_nr_pages = 0; } stock->cached = NULL; } @@ -2102,10 +2123,12 @@ static void drain_local_stock(struct work_struct *dummy) * Cache charges(val) to local per_cpu area. * This will be consumed by consume_stock() function, later. */ -static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) +static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages, + bool cache, bool kmem) { struct memcg_stock_pcp *stock; unsigned long flags; + unsigned long stock_nr_pages; local_irq_save(flags); @@ -2114,9 +2137,17 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) drain_stock(stock); stock->cached = memcg; } - stock->nr_pages += nr_pages; - if (stock->nr_pages > MEMCG_CHARGE_BATCH) + if (cache) + stock->cache_nr_pages += nr_pages; + else if (kmem) + stock->kmem_nr_pages += nr_pages; + else + stock->nr_pages += nr_pages; + + stock_nr_pages = stock->nr_pages + stock->cache_nr_pages + + stock->kmem_nr_pages; + if (nr_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); local_irq_restore(flags); @@ -2143,9 +2174,11 @@ static void drain_all_stock(struct mem_cgroup *root_memcg) for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); struct mem_cgroup *memcg; + unsigned long nr_pages = stock->nr_pages + stock->kmem_nr_pages + + stock->cache_nr_pages; memcg = stock->cached; - if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css)) + if (!memcg || !nr_pages || !css_tryget(&memcg->css)) continue; if (!mem_cgroup_is_descendant(memcg, root_memcg)) { css_put(&memcg->css); @@ -2273,17 +2306,11 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge retry: may_swap = true; kmem_limit = false; - if (consume_stock(memcg, nr_pages)) { - if (kmem_charge && !page_counter_try_charge( - &memcg->kmem, nr_pages, &counter)) { - refill_stock(memcg, nr_pages); - goto charge; - } + if (consume_stock(memcg, nr_pages, cache_charge, kmem_charge)) { css_get_many(&memcg->css, batch); goto done; } -charge: mem_over_limit = NULL; if (page_counter_try_charge(&memcg->memory, batch, &counter)) { if (do_memsw_account() && !page_counter_try_charge( @@ -2296,7 +2323,7 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge mem_over_limit = mem_cgroup_from_counter(counter, memory); if (!mem_over_limit && kmem_charge) { - if (!page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) { + if (!page_counter_try_charge(&memcg->kmem, batch, &counter)) { kmem_limit = true; mem_over_limit = mem_cgroup_from_counter(counter, kmem); page_counter_uncharge(&memcg->memory, batch); @@ -2425,13 +2452,13 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge return 0; done_restock: + if (cache_charge) + page_counter_charge(&memcg->cache, batch); + css_get_many(&memcg->css, batch); if (batch > nr_pages) - refill_stock(memcg, batch - nr_pages); + refill_stock(memcg, batch - nr_pages, cache_charge, kmem_charge); done: - if (cache_charge) - page_counter_charge(&memcg->cache, nr_pages); - /* * If the hierarchy is above the normal consumption range, schedule * reclaim on returning to userland. We can perform reclaim here @@ -6972,7 +6999,7 @@ void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages) mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); - refill_stock(memcg, nr_pages); + refill_stock(memcg, nr_pages, false, false); } static int __init cgroup_memory(char *s) -- 2.26.2 _______________________________________________ Devel mailing list Devel@openvz.org https://lists.openvz.org/mailman/listinfo/devel
