When the magazine is empty or full the zone lock is taken and a single page is operated on. This makes the zone lock hotter than it needs to be so batch allocations and frees from the zone. A larger number of pages are taken when refilling the magazine to reduce the contention on the zone->lock for IRQ-disabled callers. It's more likely that a workload will notice contention on allocations than contentions on free although of course this is workload dependant
Signed-off-by: Mel Gorman <[email protected]> --- mm/page_alloc.c | 172 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 127 insertions(+), 45 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 9ed05a5..9426174 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -601,8 +601,6 @@ static inline void __free_one_page(struct page *page, list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); out: zone->free_area[order].nr_free++; - if (unlikely(!is_migrate_isolate(migratetype))) - __mod_zone_freepage_state(zone, 1 << order, migratetype); } static inline int free_pages_check(struct page *page) @@ -634,6 +632,8 @@ static void free_one_page(struct zone *zone, struct page *page, __count_vm_events(PGFREE, 1 << order); __free_one_page(page, zone, order, migratetype); + if (unlikely(!is_migrate_isolate(migratetype))) + __mod_zone_freepage_state(zone, 1 << order, migratetype); spin_unlock_irqrestore(&zone->lock, flags); } @@ -1093,6 +1093,87 @@ void mark_free_pages(struct zone *zone) #endif /* CONFIG_PM */ #define MAGAZINE_LIMIT (1024) +#define MAGAZINE_ALLOC_BATCH (384) +#define MAGAZINE_FREE_BATCH (64) + +static +struct page *__rmqueue_magazine(struct zone *zone, int migratetype) +{ + struct page *page; + struct free_area *area = &(zone->noirq_magazine); + + if (list_empty(&area->free_list[migratetype])) + return NULL; + + /* Page is available in the magazine, allocate it */ + page = list_entry(area->free_list[migratetype].next, struct page, lru); + list_del(&page->lru); + area->nr_free--; + set_page_private(page, 0); + + return page; +} + +static void magazine_drain(struct zone *zone, int migratetype) +{ + struct free_area *area = &(zone->noirq_magazine); + struct list_head *list; + struct page *page; + unsigned int batch_free = 0; + unsigned int to_free = MAGAZINE_FREE_BATCH; + unsigned int nr_freed_cma = 0; + unsigned long flags; + LIST_HEAD(free_list); + + if (area->nr_free < MAGAZINE_LIMIT) { + spin_unlock(&zone->magazine_lock); + return; + } + + /* Free batch number of pages */ + while (to_free) { + /* + * Removes pages from lists in a round-robin fashion. A + * batch_free count is maintained that is incremented when an + * empty list is encountered. This is so more pages are freed + * off fuller lists instead of spinning excessively around empty + * lists + */ + do { + batch_free++; + if (++migratetype == MIGRATE_PCPTYPES) + migratetype = 0; + list = &area->free_list[migratetype];; + } while (list_empty(list)); + + /* This is the only non-empty list. Free them all. */ + if (batch_free == MIGRATE_PCPTYPES) + batch_free = to_free; + + do { + page = list_entry(list->prev, struct page, lru); + area->nr_free--; + set_page_private(page, 0); + list_move(&page->lru, &free_list); + if (is_migrate_isolate_page(zone, page)) + nr_freed_cma++; + } while (--to_free && --batch_free && !list_empty(list)); + } + + /* Free the list of pages to the buddy allocator */ + spin_unlock(&zone->magazine_lock); + spin_lock_irqsave(&zone->lock, flags); + while (!list_empty(&free_list)) { + page = list_entry(free_list.prev, struct page, lru); + list_del(&page->lru); + __free_one_page(page, zone, 0, get_freepage_migratetype(page)); + } + __mod_zone_page_state(zone, NR_FREE_PAGES, + MAGAZINE_FREE_BATCH - nr_freed_cma); + if (nr_freed_cma) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_freed_cma); + spin_unlock_irqrestore(&zone->lock, flags); +} /* * Free a 0-order page @@ -1111,8 +1192,10 @@ void free_hot_cold_page(struct page *page, bool cold) set_freepage_migratetype(page, migratetype); /* magazine_lock is not safe against IRQs */ - if (in_interrupt() || irqs_disabled()) - goto free_one; + if (in_interrupt() || irqs_disabled()) { + free_one_page(zone, page, 0, migratetype); + return; + } /* Put the free page on the magazine list */ spin_lock(&zone->magazine_lock); @@ -1121,32 +1204,10 @@ void free_hot_cold_page(struct page *page, bool cold) list_add(&page->lru, &area->free_list[migratetype]); else list_add_tail(&page->lru, &area->free_list[migratetype]); - page = NULL; - - /* If the magazine is full, remove a cold page for the buddy list */ - if (area->nr_free > MAGAZINE_LIMIT) { - struct list_head *list = &area->free_list[migratetype]; - int starttype = migratetype; + area->nr_free++; - while (list_empty(list)) { - if (++migratetype == MIGRATE_PCPTYPES) - migratetype = 0; - list = &area->free_list[migratetype];; - - WARN_ON_ONCE(starttype == migratetype); - } - - page = list_entry(list->prev, struct page, lru); - list_del(&page->lru); - } else { - area->nr_free++; - } - spin_unlock(&zone->magazine_lock); - -free_one: - /* Free a page back to the buddy lists if necessary */ - if (page) - free_one_page(zone, page, 0, migratetype); + /* Drain the magazine if necessary, releases the magazine lock */ + magazine_drain(zone, migratetype); } /* @@ -1261,25 +1322,46 @@ static struct page *rmqueue_magazine(struct zone *zone, int migratetype) { struct page *page = NULL; - struct free_area *area; - /* Check if it is worth acquiring the lock */ - if (!zone->noirq_magazine.nr_free) - return NULL; - - spin_lock(&zone->magazine_lock); - area = &(zone->noirq_magazine); - if (list_empty(&area->free_list[migratetype])) - goto out; + /* Only acquire the lock if there is a reasonable chance of success */ + if (zone->noirq_magazine.nr_free) { + spin_lock(&zone->magazine_lock); + page = __rmqueue_magazine(zone, migratetype); + spin_unlock(&zone->magazine_lock); + } - /* Page is available in the magazine, allocate it */ - page = list_entry(area->free_list[migratetype].next, struct page, lru); - list_del(&page->lru); - area->nr_free--; - set_page_private(page, 0); + /* Try refilling the magazine on allocaion failure */ + if (!page) { + LIST_HEAD(alloc_list); + unsigned long flags; + struct free_area *area = &(zone->noirq_magazine); + unsigned int i; + unsigned int nr_alloced = 0; + + spin_lock_irqsave(&zone->lock, flags); + for (i = 0; i < MAGAZINE_ALLOC_BATCH; i++) { + page = __rmqueue(zone, 0, migratetype); + if (!page) + break; + list_add_tail(&page->lru, &alloc_list); + nr_alloced++; + } + if (!is_migrate_cma(mt)) + __mod_zone_page_state(zone, NR_FREE_PAGES, -nr_alloced); + else + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -nr_alloced); + spin_unlock_irqrestore(&zone->lock, flags); + + spin_lock(&zone->magazine_lock); + while (!list_empty(&alloc_list)) { + page = list_entry(alloc_list.next, struct page, lru); + list_move_tail(&page->lru, &area->free_list[migratetype]); + area->nr_free++; + } + page = __rmqueue_magazine(zone, migratetype); + spin_unlock(&zone->magazine_lock); + } -out: - spin_unlock(&zone->magazine_lock); return page; } -- 1.8.1.4 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
