compound_nr_pages() exposes sparse vmemmap optimization details to the core memory initialization code.
Introduce vmemmap_nr_struct_pages() to report how many struct pages are actually allocated and need initialization for an optimized vmemmap mapping. This gives memmap_init_zone_device() the information it needs without depending on sparse-vmemmap internals. With this helper in place, drop compound_nr_pages() and keep the vmemmap-specific logic inside sparse-vmemmap code. Signed-off-by: Muchun Song <[email protected]> --- mm/internal.h | 11 ++++++++++- mm/mm_init.c | 21 +-------------------- mm/sparse.c | 13 ++++++------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 06022074ebcb..9597a703bc73 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -997,7 +997,16 @@ static inline void __section_mark_present(struct mem_section *ms, ms->section_mem_map |= SECTION_MARKED_PRESENT; } -int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages); +int vmemmap_nr_struct_pages(unsigned long pfn, unsigned long nr_pages); + +static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) +{ + VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); + VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); + + return DIV_ROUND_UP(vmemmap_nr_struct_pages(pfn, nr_pages) * sizeof(struct page), + PAGE_SIZE); +} #else static inline void sparse_memblocks_present(void) {} static inline void sparse_init(void) {} diff --git a/mm/mm_init.c b/mm/mm_init.c index 9ff118e35641..4ea39392993b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1062,25 +1062,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, } } -/* - * With compound page geometry and when struct pages are stored in ram most - * tail pages are reused. Consequently, the amount of unique struct pages to - * initialize is a lot smaller that the total amount of struct pages being - * mapped. This is a paired / mild layering violation with explicit knowledge - * of how the sparse_vmemmap internals handle compound pages in the lack - * of an altmap. - */ -static inline unsigned long compound_nr_pages(unsigned long pfn, - struct dev_pagemap *pgmap) -{ - const struct mem_section *ms = __pfn_to_section(pfn); - - if (!section_vmemmap_optimizable(ms)) - return pgmap_vmemmap_nr(pgmap); - - return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); -} - static void __ref memmap_init_compound(struct page *head, unsigned long head_pfn, unsigned long zone_idx, int nid, @@ -1145,7 +1126,7 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, pgmap)); + vmemmap_nr_struct_pages(pfn, pfns_per_compound)); } pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false, false); diff --git a/mm/sparse.c b/mm/sparse.c index 3390cb82f114..f314b9babc4a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -237,26 +237,25 @@ void __weak __meminit vmemmap_populate_print_last(void) { } -int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages) +int __meminit vmemmap_nr_struct_pages(unsigned long pfn, unsigned long nr_pages) { const unsigned int order = pfn_to_section_order(pfn); const unsigned long pages_per_compound = 1UL << order; - VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION)); - VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION); - if (!order_vmemmap_optimizable(order)) - return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE); + return nr_pages; if (order < PFN_SECTION_SHIFT) { VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound)); - return OPTIMIZED_FOLIO_VMEMMAP_PAGES * nr_pages / pages_per_compound; + return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES * nr_pages / pages_per_compound; } VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)); + /* Ensure the requested range does not cross a compound page boundary. */ + VM_WARN_ON_ONCE((pfn % pages_per_compound) + nr_pages > pages_per_compound); if (IS_ALIGNED(pfn, pages_per_compound)) - return OPTIMIZED_FOLIO_VMEMMAP_PAGES; + return OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES; return 0; } -- 2.54.0
