The function compound_nr_pages() in mm_init.c was introduced to determine how many unique struct pages need to be initialized when vmemmap optimization is enabled. However, it exposes sparse_vmemmap internals to mm_init.c.
Now that DAX and HugeTLB vmemmap optimizations are unified and simplified, we can expose a cleaner API from sparse.c to calculate the exact number of struct page structures needed. Introduce section_vmemmap_page_structs() which returns the number of page structs that require initialization, rather than the number of physical vmemmap pages to allocate. This perfectly aligns with the requirements of memmap_init_zone_device(). As a result: 1. compound_nr_pages() is removed entirely. 2. The internal section_vmemmap_pages() in sparse.c is rewritten as a simple wrapper that calculates the number of physical pages based on section_vmemmap_page_structs(). 3. A restrictive VM_BUG_ON spanning sections is removed, safely allowing compound pages (like 1G DAX pages) to cross section boundaries during device memory initialization. Signed-off-by: Muchun Song <[email protected]> --- mm/internal.h | 8 +++++++- mm/mm_init.c | 21 +-------------------- mm/sparse.c | 9 ++++----- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 7f0731e5c84f..02064f21bfe1 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -998,7 +998,13 @@ static inline void __section_mark_present(struct mem_section *ms, ms->section_mem_map |= SECTION_MARKED_PRESENT; } -int section_vmemmap_pages(unsigned long pfn, unsigned long nr_pages); +int section_vmemmap_page_structs(unsigned long pfn, unsigned long nr_pages); + +static inline int section_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) +{ + return DIV_ROUND_UP(section_vmemmap_page_structs(pfn, nr_pages) * + sizeof(struct page), PAGE_SIZE); +} #else static inline void memblocks_present(void) {} static inline void sparse_init(void) {} diff --git a/mm/mm_init.c b/mm/mm_init.c index 6b23b5f02544..74ccc556bf6e 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1060,24 +1060,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, } } -/* - * With compound page geometry and when struct pages are stored in ram most - * tail pages are reused. Consequently, the amount of unique struct pages to - * initialize is a lot smaller that the total amount of struct pages being - * mapped. This is a paired / mild layering violation with explicit knowledge - * of how the sparse_vmemmap internals handle compound pages in the lack - * of an altmap. - */ -static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, - struct dev_pagemap *pgmap, - const struct mem_section *ms) -{ - if (!section_vmemmap_optimizable(ms)) - return pgmap_vmemmap_nr(pgmap); - - return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); -} - static void __ref memmap_init_compound(struct page *head, unsigned long head_pfn, unsigned long zone_idx, int nid, @@ -1141,8 +1123,7 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(altmap, pgmap, - __pfn_to_section(pfn))); + section_vmemmap_page_structs(pfn, pfns_per_compound)); } /* diff --git a/mm/sparse.c b/mm/sparse.c index 163bb17bba96..400542302ad4 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -345,23 +345,22 @@ static void __init sparse_usage_fini(void) sparse_usagebuf = sparse_usagebuf_end = NULL; } -int __meminit section_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) +int __meminit section_vmemmap_page_structs(unsigned long pfn, unsigned long nr_pages) { const struct mem_section *ms = __pfn_to_section(pfn); unsigned int order = section_order(ms); unsigned long pages_per_compound = 1L << order; VM_BUG_ON(!IS_ALIGNED(pfn | nr_pages, min(pages_per_compound, PAGES_PER_SECTION))); - VM_BUG_ON(pfn_to_section_nr(pfn) != pfn_to_section_nr(pfn + nr_pages - 1)); if (!section_vmemmap_optimizable(ms)) - return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); + return nr_pages; if (order < PFN_SECTION_SHIFT) - return OPTIMIZED_FOLIO_VMEMMAP_PAGES * nr_pages / pages_per_compound; + return OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS * nr_pages / pages_per_compound; if (IS_ALIGNED(pfn, pages_per_compound)) - return OPTIMIZED_FOLIO_VMEMMAP_PAGES; + return OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS; return 0; } -- 2.20.1
