The commit is pushed to "branch-rh7-3.10.0-693.11.6.vz7.42.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git after rh7-3.10.0-693.11.6.vz7.42.3 ------> commit f5096f08c22af3b192b82d3904774a5dfc8e11bd Author: Kirill Tkhai <ktk...@virtuozzo.com> Date: Wed Jan 31 15:59:14 2018 +0300
tcache: Fix race between tcache_invalidate_node() and tcache_attach_page() tcache_attach_page() tcache_invalidate_node() .. __tcache_lookup_node() .. __tcache_delete_node() Check node->invalidated .. tcache_page_tree_insert() .. tcache_lru_add() .. .. tcache_invalidate_node_pages() .. node->invalidated = true Check nr_page to determ if there is a race and repeat node pages iterations if so. v2: Move invalidate assignment down in tcache_invalidate_node_tree(). v3: Synchronize sched in case of race with tcache_shrink_count() too to minimize repeats numbers. Signed-off-by: Kirill Tkhai <ktk...@virtuozzo.com> Acked-by: Andrey Ryabinin <aryabi...@virtuozzo.com> --- mm/tcache.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/mm/tcache.c b/mm/tcache.c index f1cc664a2ce3..6e0c10dd3339 100644 --- a/mm/tcache.c +++ b/mm/tcache.c @@ -676,6 +676,7 @@ static void tcache_invalidate_node(struct tcache_pool *pool, node = __tcache_lookup_node(&tree->root, key, &rb_link, &rb_parent); if (node) { tcache_hold_node(node); + node->invalidated = true; __tcache_delete_node(&tree->root, node); } spin_unlock_irq(&tree->lock); @@ -703,6 +704,7 @@ tcache_invalidate_node_tree(struct tcache_node_tree *tree) WARN_ON(atomic_read(&node->kref.refcount) != 1); WARN_ON(node->nr_pages == 0); WARN_ON(node->invalidated); + node->invalidated = true; tcache_hold_node(node); tcache_invalidate_node_pages(node); @@ -798,13 +800,16 @@ tcache_attach_page(struct tcache_node *node, pgoff_t index, struct page *page) int err = 0; tcache_init_page(page, node, index); - + /* + * Disabling of irqs implies rcu_read_lock_sched(). + * See tcache_invalidate_node_pages() for details. + */ spin_lock_irqsave(&node->tree_lock, flags); err = tcache_page_tree_insert(node, index, page); spin_unlock(&node->tree_lock); if (!err) tcache_lru_add(node->pool, page); - local_irq_restore(flags); + local_irq_restore(flags); /* Implies rcu_read_lock_sched() */ return err; } @@ -910,17 +915,16 @@ static unsigned tcache_lookup(struct page **pages, struct tcache_node *node, static noinline_for_stack void tcache_invalidate_node_pages(struct tcache_node *node) { + bool repeat, synchronize_sched_once = true; pgoff_t indices[TCACHE_PAGEVEC_SIZE]; struct page *pages[TCACHE_PAGEVEC_SIZE]; pgoff_t index = 0; unsigned nr_pages; - bool repeat; int i; /* * First forbid new page insertions - see tcache_page_tree_replace. */ - node->invalidated = true; again: repeat = false; while ((nr_pages = tcache_lookup(pages, node, index, @@ -939,6 +943,7 @@ tcache_invalidate_node_pages(struct tcache_node *node) local_irq_enable(); tcache_put_page(page); } else { + /* Race with page_ref_freeze() */ local_irq_enable(); repeat = true; } @@ -947,9 +952,18 @@ tcache_invalidate_node_pages(struct tcache_node *node) index++; } - if (repeat) { - index = 0; - goto again; + if (synchronize_sched_once) { + synchronize_sched_once = false; + if (!repeat) { + /* Race with tcache_attach_page() */ + spin_lock_irq(&node->tree_lock); + repeat = (node->nr_pages != 0); + spin_unlock_irq(&node->tree_lock); + } + if (repeat) { + synchronize_sched(); + goto again; + } } WARN_ON(node->nr_pages != 0); @@ -1202,7 +1216,7 @@ static unsigned long tcache_shrink_scan(struct shrinker *shrink, long nr_isolated, nr_reclaimed; struct page **pages; - pages = get_cpu_var(tcache_page_vec); + pages = get_cpu_var(tcache_page_vec); /* Implies rcu_read_lock_sched() */ if (WARN_ON(sc->nr_to_scan > TCACHE_SCAN_BATCH)) sc->nr_to_scan = TCACHE_SCAN_BATCH; @@ -1216,7 +1230,7 @@ static unsigned long tcache_shrink_scan(struct shrinker *shrink, if (current->reclaim_state) current->reclaim_state->reclaimed_slab += nr_reclaimed; out: - put_cpu_var(tcache_page_vec); + put_cpu_var(tcache_page_vec); /* Implies rcu_read_unlock_sched() */ return nr_reclaimed; } _______________________________________________ Devel mailing list Devel@openvz.org https://lists.openvz.org/mailman/listinfo/devel