free_base_page_list() frees a list of pages. This patch will batch the magazine lock for the list of pages if possible.
Signed-off-by: Mel Gorman <[email protected]> --- mm/page_alloc.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 727c8d3..cf31191 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1236,15 +1236,13 @@ static void magazine_drain(struct zone *zone, struct free_magazine *mag, spin_unlock_irqrestore(&zone->lock, flags); } -/* Free a 0-order page */ -void free_base_page(struct page *page) +/* Prepare a page for freeing and return its migratetype */ +static inline int free_base_page_prep(struct page *page) { - struct zone *zone = page_zone(page); - struct free_magazine *mag; int migratetype; if (!free_pages_prepare(page, 0)) - return; + return -1; migratetype = get_pageblock_migratetype(page); @@ -1256,24 +1254,46 @@ void free_base_page(struct page *page) * excessively into the page allocator */ if (migratetype >= MIGRATE_PCPTYPES) { - if (unlikely(is_migrate_isolate(migratetype))) { - free_one_page(zone, page, 0, migratetype); - return; - } - migratetype = MIGRATE_MOVABLE; + if (likely(!is_migrate_isolate(migratetype))) + migratetype = MIGRATE_MOVABLE; } + + set_freepage_migratetype(page, migratetype); + + return migratetype; +} + +/* Put the free page on the magazine list with magazine lock held */ +static inline void __free_base_page(struct zone *zone, + struct free_area_magazine *area, + struct page *page, int migratetype) +{ + list_add(&page->lru, &area->free_list[migratetype]); + area->nr_free++; +} + +/* Free a 0-order page */ +void free_base_page(struct page *page) +{ + struct zone *zone = page_zone(page); + struct free_magazine *mag; + int migratetype; + + migratetype = free_base_page_prep(page); + if (migratetype == -1) + return; set_freepage_migratetype(page, migratetype); /* magazine_lock is not safe against IRQs */ - if (in_interrupt() || irqs_disabled()) { + if (migratetype >= MIGRATE_PCPTYPES || in_interrupt() || + irqs_disabled()) { free_one_page(zone, page, 0, migratetype); return; } /* Put the free page on the magazine list */ mag = lock_magazine(zone); - list_add(&page->lru, &mag->area.free_list[migratetype]); - mag->area.nr_free++; + __free_base_page(zone, &mag->area, page, migratetype); /* Drain the magazine if necessary, releases the magazine lock */ magazine_drain(zone, mag, migratetype); @@ -1283,11 +1303,38 @@ void free_base_page(struct page *page) void free_base_page_list(struct list_head *list) { struct page *page, *next; + struct zone *locked_zone = NULL; + struct free_magazine *mag = NULL; + bool use_magazine = (!in_interrupt() && !irqs_disabled()); + int migratetype = MIGRATE_UNMOVABLE; + /* Similar to free_hot_cold_page except magazine lock is batched */ list_for_each_entry_safe(page, next, list, lru) { + struct zone *zone = page_zone(page); + int migratetype; + trace_mm_page_free_batched(page); - free_base_page(page); + migratetype = free_base_page_prep(page); + if (migratetype == -1) + continue; + + if (!use_magazine || migratetype >= MIGRATE_PCPTYPES) { + free_one_page(zone, page, 0, migratetype); + continue; + } + + if (zone != locked_zone) { + /* Drain unlocks magazine lock */ + if (locked_zone) + magazine_drain(locked_zone, mag, migratetype); + mag = lock_magazine(zone); + locked_zone = zone; + } + __free_base_page(zone, &mag->area, page, migratetype); } + + if (locked_zone) + magazine_drain(locked_zone, mag, migratetype); } /* -- 1.8.1.4 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
