When free page reporting unaccepts isolated pages on confidential computing guests, __putback_isolated_page() must not return those unaccepted pages directly to zone->free_area; doing so would violate the allocator invariant that all pages on zone->free_area are accepted.
Update __putback_isolated_page(), __free_pages_ok(), __free_frozen_pages(), and free_unref_folios() to route dynamically unaccepted pages to zone->unaccepted_pages via __free_unaccepted_locked() (falling back to immediate acceptance for isolated or CMA pageblocks). In addition: - Extract free_pages_prepare_init() so post-acceptance poisoning, zeroing, KASAN tagging, arch_free_page(), and debug_pagealloc unmapping run after accept_memory() without touching unaccepted pages in __free_pages_prepare() or re-running metadata teardown in __accept_page(). - Pass FPI_SKIP_REPORT_NOTIFY when __accept_page() frees a freshly accepted block into the buddy allocator so it is not immediately re-queued for free page reporting before allocation. Signed-off-by: Kevin Loughlin <[email protected]> --- mm/page_alloc.c | 120 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 35 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 46ae8c6f3814..c10b77304633 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1298,12 +1298,53 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #endif /* CONFIG_MEM_ALLOC_PROFILING */ +static inline bool should_free_unaccepted(struct page *page, + unsigned int order, + fpi_t fpi_flags) +{ + return !(fpi_flags & FPI_NOLOCK) && + page_contains_unaccepted(page, order); +} + +static void free_pages_prepare_init(struct page *page, unsigned int order) +{ + bool skip_kasan_poison = should_skip_kasan_poison(page); + bool init = want_init_on_free(); + + kernel_poison_pages(page, 1 << order); + + /* + * As memory initialization might be integrated into KASAN, + * KASAN poisoning and memory initialization code must be + * kept together to avoid discrepancies in behavior. + * + * With hardware tag-based KASAN, memory tags must be set before the + * page becomes unavailable via debug_pagealloc or arch_free_page. + */ + if (!skip_kasan_poison) { + kasan_poison_pages(page, order, init); + + /* Memory is already initialized if KASAN did it internally. */ + if (kasan_has_integrated_init()) + init = false; + } + if (init) + clear_highpages_kasan_tagged(page, 1 << order); + + /* + * arch_free_page() can make the page's contents inaccessible. s390 + * does this. So nothing which can access the page's contents should + * happen after this. + */ + arch_free_page(page, order); + + debug_pagealloc_unmap_pages(page, 1 << order); +} + static __always_inline bool __free_pages_prepare(struct page *page, unsigned int order, fpi_t fpi_flags) { int bad = 0; - bool skip_kasan_poison = should_skip_kasan_poison(page); - bool init = want_init_on_free(); bool compound = PageCompound(page); struct folio *folio = page_folio(page); @@ -1411,34 +1452,10 @@ static __always_inline bool __free_pages_prepare(struct page *page, PAGE_SIZE << order); } - kernel_poison_pages(page, 1 << order); - - /* - * As memory initialization might be integrated into KASAN, - * KASAN poisoning and memory initialization code must be - * kept together to avoid discrepancies in behavior. - * - * With hardware tag-based KASAN, memory tags must be set before the - * page becomes unavailable via debug_pagealloc or arch_free_page. - */ - if (!skip_kasan_poison) { - kasan_poison_pages(page, order, init); - - /* Memory is already initialized if KASAN did it internally. */ - if (kasan_has_integrated_init()) - init = false; - } - if (init) - clear_highpages_kasan_tagged(page, 1 << order); - - /* - * arch_free_page() can make the page's contents inaccessible. s390 - * does this. So nothing which can access the page's contents should - * happen after this. - */ - arch_free_page(page, order); + if (should_free_unaccepted(page, order, fpi_flags)) + return true; - debug_pagealloc_unmap_pages(page, 1 << order); + free_pages_prepare_init(page, order); return true; } @@ -1572,14 +1589,31 @@ static void free_one_page(struct zone *zone, struct page *page, __count_vm_events(PGFREE, 1 << order); } +static bool handle_unaccepted_free(struct page *page, unsigned int order, + fpi_t fpi_flags) +{ + if (!should_free_unaccepted(page, order, fpi_flags)) + return false; + + if (__free_unaccepted(page, order)) + return true; + + accept_memory(page_to_phys(page), PAGE_SIZE << order); + free_pages_prepare_init(page, order); + return false; +} + static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags) { unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); - if (__free_pages_prepare(page, order, fpi_flags)) + if (__free_pages_prepare(page, order, fpi_flags)) { + if (handle_unaccepted_free(page, order, fpi_flags)) + return; free_one_page(zone, page, pfn, order, fpi_flags); + } } void __meminit __free_pages_core(struct page *page, unsigned int order, @@ -2955,7 +2989,8 @@ static void __free_frozen_pages(struct page *page, unsigned int order, unsigned long pfn = page_to_pfn(page); int migratetype; - if (!pcp_allowed_order(order)) { + if (!pcp_allowed_order(order) || + should_free_unaccepted(page, order, fpi_flags)) { __free_pages_ok(page, order, fpi_flags); return; } @@ -3024,9 +3059,13 @@ void free_unref_folios(struct folio_batch *folios) continue; /* * Free orders not handled on the PCP directly to the - * allocator. + * allocator, as well as any pages containing unaccepted memory. */ - if (!pcp_allowed_order(order)) { + if (!pcp_allowed_order(order) || + should_free_unaccepted(&folio->page, order, FPI_NONE)) { + if (handle_unaccepted_free(&folio->page, order, + FPI_NONE)) + continue; free_one_page(folio_zone(folio), &folio->page, pfn, order, FPI_NONE); continue; @@ -3187,6 +3226,16 @@ void __putback_isolated_page(struct page *page, unsigned int order, int mt) /* zone lock should be held when this function is called */ lockdep_assert_held(&zone->lock); + if (page_contains_unaccepted(page, order)) { + if (__free_unaccepted_locked(page, zone, order)) + return; + accept_memory(page_to_phys(page), PAGE_SIZE << order); + free_pages_prepare_init(page, order); + split_large_buddy(zone, page, page_to_pfn(page), order, + FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); + return; + } + /* Return isolated page to tail of freelist. */ __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); @@ -7933,8 +7982,9 @@ static void __accept_page(struct zone *zone, unsigned long *flags, spin_unlock_irqrestore(&zone->lock, *flags); accept_memory(page_to_phys(page), PAGE_SIZE << order); - - __free_pages_ok(page, order, FPI_TO_TAIL); + free_pages_prepare_init(page, order); + free_one_page(zone, page, page_to_pfn(page), order, + FPI_TO_TAIL | FPI_SKIP_REPORT_NOTIFY); } void accept_page(struct page *page) -- 2.56.0.307.g5fe7bd680c-goog
