Each zone's free_area tracks the number of free pages for all free lists. This does not allow the number of free pages for a specific migratetype to be determined without iterating its free list.
An upcoming change will use this information to preclude doing async memory compaction when the number of MIGRATE_UNMOVABLE pageblocks is below a certain threshold. The total number of free pages is still tracked, however, to not make zone_watermark_ok() more expensive. Reading /proc/pagetypeinfo, however, is faster. This patch introduces no functional change and increases the amount of per-zone metadata at worst by 48 bytes per memory zone (when CONFIG_CMA and CONFIG_MEMORY_ISOLATION are enabled). Signed-off-by: David Rientjes <[email protected]> --- include/linux/mmzone.h | 3 ++- mm/compaction.c | 4 ++-- mm/page_alloc.c | 47 ++++++++++++++++++++++++++++------------------- mm/vmstat.c | 18 +++++------------- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -89,7 +89,8 @@ extern int page_group_by_mobility_disabled; struct free_area { struct list_head free_list[MIGRATE_TYPES]; - unsigned long nr_free; + unsigned long nr_free[MIGRATE_TYPES]; + unsigned long total_free; }; struct pglist_data; diff --git a/mm/compaction.c b/mm/compaction.c --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1320,13 +1320,13 @@ static enum compact_result __compact_finished(struct zone *zone, struct compact_ bool can_steal; /* Job done if page is free of the right migratetype */ - if (!list_empty(&area->free_list[migratetype])) + if (area->nr_free[migratetype]) return COMPACT_SUCCESS; #ifdef CONFIG_CMA /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */ if (migratetype == MIGRATE_MOVABLE && - !list_empty(&area->free_list[MIGRATE_CMA])) + area->nr_free[MIGRATE_CMA]) return COMPACT_SUCCESS; #endif /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -821,7 +821,8 @@ static inline void __free_one_page(struct page *page, clear_page_guard(zone, buddy, order, migratetype); } else { list_del(&buddy->lru); - zone->free_area[order].nr_free--; + zone->free_area[order].nr_free[migratetype]--; + zone->free_area[order].total_free--; rmv_page_order(buddy); } combined_idx = buddy_idx & page_idx; @@ -880,7 +881,8 @@ static inline void __free_one_page(struct page *page, list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); out: - zone->free_area[order].nr_free++; + zone->free_area[order].nr_free[migratetype]++; + zone->free_area[order].total_free++; } /* @@ -1648,7 +1650,8 @@ static inline void expand(struct zone *zone, struct page *page, continue; list_add(&page[size].lru, &area->free_list[migratetype]); - area->nr_free++; + area->nr_free[migratetype]++; + area->total_free++; set_page_order(&page[size], high); } } @@ -1802,7 +1805,8 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, continue; list_del(&page->lru); rmv_page_order(page); - area->nr_free--; + area->nr_free[migratetype]--; + area->total_free--; expand(zone, page, order, current_order, area, migratetype); set_pcppage_migratetype(page, migratetype); return page; @@ -1991,7 +1995,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order, int i; int fallback_mt; - if (area->nr_free == 0) + if (!area->total_free) return -1; *can_steal = false; @@ -2000,7 +2004,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order, if (fallback_mt == MIGRATE_TYPES) break; - if (list_empty(&area->free_list[fallback_mt])) + if (!area->nr_free[fallback_mt]) continue; if (can_steal_fallback(order, migratetype)) @@ -2163,7 +2167,8 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) steal_suitable_fallback(zone, page, start_migratetype); /* Remove the page from the freelists */ - area->nr_free--; + area->nr_free[fallback_mt]--; + area->total_free--; list_del(&page->lru); rmv_page_order(page); @@ -2549,7 +2554,8 @@ int __isolate_free_page(struct page *page, unsigned int order) /* Remove page from free list */ list_del(&page->lru); - zone->free_area[order].nr_free--; + zone->free_area[order].nr_free[mt]--; + zone->free_area[order].total_free--; rmv_page_order(page); /* @@ -2808,22 +2814,19 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, struct free_area *area = &z->free_area[o]; int mt; - if (!area->nr_free) + if (!area->total_free) continue; if (alloc_harder) return true; - for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { - if (!list_empty(&area->free_list[mt])) + for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) + if (area->nr_free[mt]) return true; - } #ifdef CONFIG_CMA - if ((alloc_flags & ALLOC_CMA) && - !list_empty(&area->free_list[MIGRATE_CMA])) { + if ((alloc_flags & ALLOC_CMA) && area->nr_free[MIGRATE_CMA]) return true; - } #endif } return false; @@ -4431,12 +4434,12 @@ void show_free_areas(unsigned int filter) struct free_area *area = &zone->free_area[order]; int type; - nr[order] = area->nr_free; + nr[order] = area->total_free; total += nr[order] << order; types[order] = 0; for (type = 0; type < MIGRATE_TYPES; type++) { - if (!list_empty(&area->free_list[type])) + if (area->nr_free[type]) types[order] |= 1 << type; } } @@ -5100,8 +5103,10 @@ static void __meminit zone_init_free_lists(struct zone *zone) unsigned int order, t; for_each_migratetype_order(order, t) { INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); - zone->free_area[order].nr_free = 0; + zone->free_area[order].nr_free[t] = 0; } + for (order = 0; order < MAX_ORDER; order++) + zone->free_area[order].total_free = 0; } #ifndef __HAVE_ARCH_MEMMAP_INIT @@ -7416,6 +7421,8 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) spin_lock_irqsave(&zone->lock, flags); pfn = start_pfn; while (pfn < end_pfn) { + int migratetype; + if (!pfn_valid(pfn)) { pfn++; continue; @@ -7438,9 +7445,11 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) pr_info("remove from free list %lx %d %lx\n", pfn, 1 << order, end_pfn); #endif + migratetype = get_pageblock_migratetype(page); list_del(&page->lru); rmv_page_order(page); - zone->free_area[order].nr_free--; + zone->free_area[order].nr_free[migratetype]--; + zone->free_area[order].total_free--; for (i = 0; i < (1 << order); i++) SetPageReserved((page+i)); pfn += (1 << order); diff --git a/mm/vmstat.c b/mm/vmstat.c --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -846,7 +846,7 @@ static void fill_contig_page_info(struct zone *zone, unsigned long blocks; /* Count number of free blocks */ - blocks = zone->free_area[order].nr_free; + blocks = zone->free_area[order].total_free; info->free_blocks_total += blocks; /* Count free base pages */ @@ -1146,7 +1146,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (order = 0; order < MAX_ORDER; ++order) - seq_printf(m, "%6lu ", zone->free_area[order].nr_free); + seq_printf(m, "%6lu ", zone->free_area[order].total_free); seq_putc(m, '\n'); } @@ -1170,17 +1170,9 @@ static void pagetypeinfo_showfree_print(struct seq_file *m, pgdat->node_id, zone->name, migratetype_names[mtype]); - for (order = 0; order < MAX_ORDER; ++order) { - unsigned long freecount = 0; - struct free_area *area; - struct list_head *curr; - - area = &(zone->free_area[order]); - - list_for_each(curr, &area->free_list[mtype]) - freecount++; - seq_printf(m, "%6lu ", freecount); - } + for (order = 0; order < MAX_ORDER; ++order) + seq_printf(m, "%6lu ", + zone->free_area[order].nr_free[mtype]); seq_putc(m, '\n'); } }
