Currently, zone->unaccepted_pages assumes all unaccepted blocks are MAX_PAGE_ORDER. However, dynamic memory unacceptance operates at unaccepted_memory_order() granularity (typically 2 MiB / PMD_ORDER on x86).
Introduce has_unaccepted_memory() to report whether lazy unaccepted memory and dynamic unacceptance are active (returning false when booted with accept_memory=eager), and generalize zone->unaccepted_pages from MAX_PAGE_ORDER to orders >= unaccepted_memory_order() stored in page_private(page): - Update __free_unaccepted_locked() to verify that pageblocks are mergeable (moving blocks >= pageblock_order to MIGRATE_MOVABLE) and record the block order in page_private(page). - Update __accept_page(), try_to_accept_memory_one(), and cond_accept_memory() to accept blocks of order page_private(page), and ensure cond_accept_memory() accepts at least (1 << order) pages when high-order watermark or rmqueue checks fail. - Update prep_move_freepages_block(), set_migratetype_isolate(), isolate_single_pageblock(), and skip_buddy_pages() to handle variable-order PageUnaccepted blocks. Signed-off-by: Kevin Loughlin <[email protected]> --- Documentation/mm/physical_memory.rst | 3 +- drivers/firmware/efi/unaccepted_memory.c | 2 +- include/linux/mm.h | 6 ++ include/linux/mmzone.h | 2 +- mm/page_alloc.c | 127 ++++++++++++++++++----- mm/page_isolation.c | 44 +++++++- mm/page_owner.c | 7 +- 7 files changed, 155 insertions(+), 36 deletions(-) diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst index 2e67e8b23a99..e9fb554932d5 100644 --- a/Documentation/mm/physical_memory.rst +++ b/Documentation/mm/physical_memory.rst @@ -526,7 +526,8 @@ General ``zone_init_free_lists()``. ``unaccepted_pages`` - The list of pages to be accepted. All pages on the list are ``MAX_PAGE_ORDER``. + The list of pages to be accepted. All pages on the list are at least + ``unaccepted_memory_order()`` and record their order in ``page_private()``. Defined only when ``CONFIG_UNACCEPTED_MEMORY`` is enabled. ``flags`` diff --git a/drivers/firmware/efi/unaccepted_memory.c b/drivers/firmware/efi/unaccepted_memory.c index 128bc1b113cd..769faf430690 100644 --- a/drivers/firmware/efi/unaccepted_memory.c +++ b/drivers/firmware/efi/unaccepted_memory.c @@ -246,7 +246,7 @@ int mark_memory_unaccepted(phys_addr_t start, unsigned long size) unsigned long flags; u64 unit_size; - if (!arch_has_unaccept_memory()) + if (!has_unaccepted_memory()) return -EOPNOTSUPP; unaccepted = efi_get_unaccepted_table(); diff --git a/include/linux/mm.h b/include/linux/mm.h index 27bb40651c85..619c71565dd6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5362,6 +5362,7 @@ int set_anon_vma_name(unsigned long addr, unsigned long size, bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size); void accept_memory(phys_addr_t start, unsigned long size); +bool has_unaccepted_memory(void); unsigned int unaccepted_memory_order(void); int mark_memory_unaccepted(phys_addr_t start, unsigned long size); @@ -5377,6 +5378,11 @@ static inline void accept_memory(phys_addr_t start, unsigned long size) { } +static inline bool has_unaccepted_memory(void) +{ + return false; +} + static inline unsigned int unaccepted_memory_order(void) { return 0; diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index d44b04589f48..ae4657dbb5a8 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1112,7 +1112,7 @@ struct zone { struct free_area free_area[NR_PAGE_ORDERS]; #ifdef CONFIG_UNACCEPTED_MEMORY - /* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */ + /* Pages to be accepted. All pages are >= unaccepted_memory_order() */ struct list_head unaccepted_pages; /* To be called once the last page in the zone is accepted */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3322f9e75616..46ae8c6f3814 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -300,8 +300,10 @@ static unsigned long alloc_stall_warn_jiffies = INITIAL_JIFFIES; static bool page_contains_unaccepted(struct page *page, unsigned int order); static bool cond_accept_memory(struct zone *zone, unsigned int order, - int alloc_flags); -static bool __free_unaccepted(struct page *page); + int alloc_flags, bool force); +static bool __free_unaccepted(struct page *page, unsigned int order); +static bool __free_unaccepted_locked(struct page *page, struct zone *zone, + unsigned int order); int page_group_by_mobility_disabled __read_mostly; @@ -1614,8 +1616,10 @@ void __meminit __free_pages_core(struct page *page, unsigned int order, atomic_long_add(nr_pages, &page_zone(page)->managed_pages); } - if (page_contains_unaccepted(page, order)) { - if (order == MAX_PAGE_ORDER && __free_unaccepted(page)) + if (range_contains_unaccepted_memory(page_to_phys(page), + PAGE_SIZE << order)) { + if (order >= unaccepted_memory_order() && + __free_unaccepted(page, order)) return; accept_memory(page_to_phys(page), PAGE_SIZE << order); @@ -2003,6 +2007,13 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page, pfn += nr; continue; } + if (PageUnaccepted(page)) { + int nr = 1 << page_private(page); + + *num_free += nr; + pfn += nr; + continue; + } /* * We assume that pages that could be isolated for * migration are movable. But we don't actually try @@ -3872,7 +3883,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, continue; } - cond_accept_memory(zone, order, alloc_flags); + cond_accept_memory(zone, order, alloc_flags, false); /* * Detect whether the number of free pages is below high @@ -3897,7 +3908,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, if (!zone_watermark_fast(zone, order, mark, ac->highest_zoneidx, alloc_flags, gfp_mask)) { - if (cond_accept_memory(zone, order, alloc_flags)) + if (cond_accept_memory(zone, order, alloc_flags, true)) goto try_this_zone; /* @@ -3934,7 +3945,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, return page; } else { - if (cond_accept_memory(zone, order, alloc_flags)) + if (cond_accept_memory(zone, order, alloc_flags, true)) goto try_this_zone; /* Try again if zone has deferred pages */ @@ -5216,7 +5227,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, goto failed; } - cond_accept_memory(zone, 0, alloc_flags); + cond_accept_memory(zone, 0, alloc_flags, false); retry_this_zone: mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages - nr_populated; if (zone_watermark_fast(zone, 0, mark, @@ -5225,7 +5236,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, break; } - if (cond_accept_memory(zone, 0, alloc_flags)) + if (cond_accept_memory(zone, 0, alloc_flags, true)) goto retry_this_zone; /* Try again if zone has deferred pages */ @@ -7867,6 +7878,7 @@ bool has_managed_zone(enum zone_type zone) } #ifdef CONFIG_UNACCEPTED_MEMORY +#include <asm/unaccepted_memory.h> static bool lazy_accept = true; @@ -7884,25 +7896,45 @@ static int __init accept_memory_parse(char *p) } early_param("accept_memory", accept_memory_parse); +bool has_unaccepted_memory(void) +{ + return lazy_accept && efi_get_unaccepted_table() && + arch_has_unaccept_memory() && + unaccepted_memory_order() <= MAX_PAGE_ORDER; +} +EXPORT_SYMBOL_GPL(has_unaccepted_memory); + static bool page_contains_unaccepted(struct page *page, unsigned int order) { phys_addr_t start = page_to_phys(page); + /* + * Unaccepted memory is tracked and freed in blocks of at least + * unaccepted_memory_order(). Skipping lower-order frees avoids + * acquiring unaccepted_memory_lock on normal page frees. + */ + if (!has_unaccepted_memory() || order < unaccepted_memory_order()) + return false; + return range_contains_unaccepted_memory(start, PAGE_SIZE << order); } static void __accept_page(struct zone *zone, unsigned long *flags, struct page *page) { + unsigned int order = page_private(page); + unsigned int nr_pages = 1 << order; + list_del(&page->lru); - account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); - __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); + account_freepages(zone, -nr_pages, MIGRATE_MOVABLE); + __mod_zone_page_state(zone, NR_UNACCEPTED, -nr_pages); __ClearPageUnaccepted(page); + set_page_private(page, 0); spin_unlock_irqrestore(&zone->lock, *flags); - accept_memory(page_to_phys(page), PAGE_SIZE << MAX_PAGE_ORDER); + accept_memory(page_to_phys(page), PAGE_SIZE << order); - __free_pages_ok(page, MAX_PAGE_ORDER, FPI_TO_TAIL); + __free_pages_ok(page, order, FPI_TO_TAIL); } void accept_page(struct page *page) @@ -7920,27 +7952,29 @@ void accept_page(struct page *page) __accept_page(zone, &flags, page); } -static bool try_to_accept_memory_one(struct zone *zone) +static unsigned int try_to_accept_memory_one(struct zone *zone) { unsigned long flags; struct page *page; + unsigned int nr_pages; spin_lock_irqsave(&zone->lock, flags); page = list_first_entry_or_null(&zone->unaccepted_pages, struct page, lru); if (!page) { spin_unlock_irqrestore(&zone->lock, flags); - return false; + return 0; } + nr_pages = 1 << page_private(page); /* Unlocks zone->lock */ __accept_page(zone, &flags, page); - return true; + return nr_pages; } static bool cond_accept_memory(struct zone *zone, unsigned int order, - int alloc_flags) + int alloc_flags, bool force) { long to_accept, wmark; bool ret = false; @@ -7957,7 +7991,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, /* * Watermarks have not been initialized yet. * - * Accepting one MAX_PAGE_ORDER page to ensure progress. + * Accepting one block to ensure progress. */ if (!wmark) return try_to_accept_memory_one(zone); @@ -7968,32 +8002,62 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order, __zone_watermark_unusable_free(zone, order, 0) - zone_page_state(zone, NR_UNACCEPTED)); + if (force && to_accept < (1L << order)) + to_accept = 1L << order; + while (to_accept > 0) { - if (!try_to_accept_memory_one(zone)) + unsigned int accepted = try_to_accept_memory_one(zone); + + if (!accepted) break; ret = true; - to_accept -= MAX_ORDER_NR_PAGES; + to_accept -= accepted; } return ret; } -static bool __free_unaccepted(struct page *page) +static bool __free_unaccepted_locked(struct page *page, struct zone *zone, + unsigned int order) +{ + unsigned int nr_pages = 1 << order; + unsigned int step = max_t(unsigned int, 1, pageblock_nr_pages); + unsigned int i; + + lockdep_assert_held(&zone->lock); + + for (i = 0; i < nr_pages; i += step) { + int mt = get_pageblock_migratetype(page + i); + + if (!migratetype_is_mergeable(mt)) + return false; + } + + if (order >= pageblock_order) + change_pageblock_range(page, order, MIGRATE_MOVABLE); + + set_page_private(page, order); + list_add_tail(&page->lru, &zone->unaccepted_pages); + account_freepages(zone, nr_pages, MIGRATE_MOVABLE); + __mod_zone_page_state(zone, NR_UNACCEPTED, nr_pages); + __SetPageUnaccepted(page); + return true; +} + +static bool __free_unaccepted(struct page *page, unsigned int order) { struct zone *zone = page_zone(page); unsigned long flags; + bool ret; if (!lazy_accept) return false; spin_lock_irqsave(&zone->lock, flags); - list_add_tail(&page->lru, &zone->unaccepted_pages); - account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); - __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); - __SetPageUnaccepted(page); + ret = __free_unaccepted_locked(page, zone, order); spin_unlock_irqrestore(&zone->lock, flags); - return true; + return ret; } #else @@ -8004,12 +8068,19 @@ static bool page_contains_unaccepted(struct page *page, unsigned int order) } static bool cond_accept_memory(struct zone *zone, unsigned int order, - int alloc_flags) + int alloc_flags, bool force) { return false; } -static bool __free_unaccepted(struct page *page) +static bool __free_unaccepted_locked(struct page *page, struct zone *zone, + unsigned int order) +{ + BUILD_BUG(); + return false; +} + +static bool __free_unaccepted(struct page *page, unsigned int order) { BUILD_BUG(); return false; diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 5aaad037384e..c85ce158a646 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -165,6 +165,36 @@ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long e return NULL; } +static struct page *find_unaccepted_page_in_pageblock(struct zone *zone, + struct page *page) +{ +#ifdef CONFIG_UNACCEPTED_MEMORY + unsigned long pb_start = pageblock_start_pfn(page_to_pfn(page)); + unsigned long pb_end = pageblock_end_pfn(page_to_pfn(page)); + unsigned long start_pfn, pfn, step; + + lockdep_assert_held(&zone->lock); + + if (list_empty(&zone->unaccepted_pages) || + unaccepted_memory_order() > MAX_PAGE_ORDER) + return NULL; + + start_pfn = ALIGN_DOWN(pb_start, MAX_ORDER_NR_PAGES); + step = 1UL << unaccepted_memory_order(); + + for (pfn = start_pfn; pfn < pb_end; pfn += step) { + struct page *cand = pfn_to_online_page(pfn); + + if (!cand || page_zone(cand) != zone || !PageUnaccepted(cand)) + continue; + + if (pfn + (1UL << page_private(cand)) > pb_start) + return cand; + } +#endif + return NULL; +} + /* * This function set pageblock migratetype to isolate if no unmovable page is * present in [start_pfn, end_pfn). The pageblock must intersect with @@ -175,10 +205,14 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode, { struct zone *zone = page_zone(page); struct page *unmovable; + struct page *unaccepted = NULL; unsigned long check_unmovable_start, check_unmovable_end; - if (PageUnaccepted(page)) - accept_page(page); +retry: + if (unaccepted) { + accept_page(unaccepted); + unaccepted = NULL; + } scoped_guard(spinlock_irqsave, &zone->lock) { /* @@ -189,6 +223,10 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode, if (is_migrate_isolate_page(page)) return -EBUSY; + unaccepted = find_unaccepted_page_in_pageblock(zone, page); + if (unaccepted) + goto retry; + /* * FIXME: Now, memory hotplug doesn't call shrink_slab() by * itself. We just check MOVABLE pages. @@ -383,7 +421,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, pfn = page_to_pfn(page); if (PageUnaccepted(page)) { - pfn += MAX_ORDER_NR_PAGES; + accept_page(page); continue; } diff --git a/mm/page_owner.c b/mm/page_owner.c index cfc31c92d765..e3f3a019b0bc 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -459,10 +459,13 @@ static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) { unsigned long order; - if (!PageBuddy(page)) + if (PageBuddy(page)) + order = buddy_order_unsafe(page); + else if (PageUnaccepted(page)) + order = READ_ONCE(page->private); + else return false; - order = buddy_order_unsafe(page); if (order <= MAX_PAGE_ORDER) { unsigned long new_pfn = *pfn + (1UL << order); unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); -- 2.56.0.307.g5fe7bd680c-goog
