The vmemmap optimization helpers currently live in mm/sparse.h, which is an internal MM header. That works for MM code, but prevents powerpc from using the same interfaces without including a private header.
Move the declarations and inline helpers to vmemmap-optimization.h. This is a preparatory change for powerpc, which has its own vmemmap optimization implementation and needs to use the common vmemmap optimization interfaces from architecture code. Signed-off-by: Muchun Song <[email protected]> Acked-by: Qi Zheng <[email protected]> Acked-by: Mike Rapoport (Microsoft) <[email protected]> --- v5: - Move the VMEMMAP_OPTIMIZATION_* macros and MAX_FOLIO_VMEMMAP_ALIGN to vmemmap-optimization.h v4: - Use the renamed VMEMMAP_OPTIMIZATION config in the public header - Collect Acked-by from Mike Rapoport v3: - Update the subject and commit message to describe common vmemmap optimization helpers - Collect Acked-by from Qi Zheng v2: - Fix missing header dependencies. --- MAINTAINERS | 1 + arch/loongarch/include/asm/pgtable.h | 1 + arch/riscv/mm/init.c | 1 + include/linux/mmzone.h | 17 ----- include/linux/vmemmap-optimization.h | 109 +++++++++++++++++++++++++++ mm/hugetlb.c | 2 +- mm/hugetlb_vmemmap.c | 2 +- mm/sparse.h | 78 +------------------ 8 files changed, 115 insertions(+), 96 deletions(-) create mode 100644 include/linux/vmemmap-optimization.h diff --git a/MAINTAINERS b/MAINTAINERS index 7eae9228f547..6aa7308cb6c3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12096,6 +12096,7 @@ F: Documentation/mm/hugetlbfs_reserv.rst F: Documentation/mm/vmemmap_dedup.rst F: fs/hugetlbfs/ F: include/linux/hugetlb.h +F: include/linux/vmemmap-optimization.h F: include/trace/events/hugetlbfs.h F: mm/hugetlb.c F: mm/hugetlb_cgroup.c diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index cf29a4c8ac59..f87603135131 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -72,6 +72,7 @@ #include <linux/mm_types.h> #include <linux/mmzone.h> +#include <linux/vmemmap-optimization.h> #include <asm/fixmap.h> #include <asm/sparsemem.h> diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efe..857f9a55039c 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -22,6 +22,7 @@ #include <linux/hugetlb.h> #include <linux/kfence.h> #include <linux/execmem.h> +#include <linux/vmemmap-optimization.h> #include <asm/alternative.h> #include <asm/fixmap.h> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index cd68c1904c91..65de3bb13eb3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -96,23 +96,6 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - -/* The number of retained vmemmap pages with HVO enabled. */ -#define VMEMMAP_OPTIMIZATION_PAGES 1 -#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ - (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) -#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) - enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h new file mode 100644 index 000000000000..bd0974b262a4 --- /dev/null +++ b/include/linux/vmemmap-optimization.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * vmemmap-optimization.h + * + * Generic vmemmap optimization declarations. + * + * Author: Muchun Song <[email protected]> + */ +#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H +#define _LINUX_VMEMMAP_OPTIMIZATION_H + +#include <linux/align.h> +#include <linux/log2.h> +#include <linux/mmdebug.h> +#include <linux/mmzone.h> + +/* + * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + * + * HVO which is only active if the size of struct page is a power of 2. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ + is_power_of_2(sizeof(struct page)) ? \ + MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) + +/* The number of retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_PAGES 1 +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ + (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) +#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) + +#ifdef CONFIG_VMEMMAP_OPTIMIZATION +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return section->compound_page_order; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ + VM_WARN_ON(section_compound_order(section) && order && + section_compound_order(section) != order); + section->compound_page_order = order; +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ + unsigned long section_nr = pfn_to_section_nr(pfn); + + if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) + return; + + for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) + section_set_compound_order(__nr_to_section(section_nr + i), order); +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return section_compound_order(__pfn_to_section(pfn)); +} + +struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); +#else +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return 0; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return 0; +} +#endif /* CONFIG_VMEMMAP_OPTIMIZATION */ + +static inline bool vmemmap_optimizable_pfn(unsigned long pfn) +{ + const unsigned int order = pfn_to_section_compound_order(pfn); + const unsigned long nr_pages = 1UL << order; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; +} + +static inline bool vmemmap_optimizable_order(unsigned int order) +{ + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; +} +#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index fd00141b089a..2003439ea13c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -38,6 +38,7 @@ #include <linux/mm_inline.h> #include <linux/padata.h> #include <linux/pgalloc.h> +#include <linux/vmemmap-optimization.h> #include <asm/page.h> #include <asm/tlb.h> @@ -52,7 +53,6 @@ #include "hugetlb_cma.h" #include "hugetlb_internal.h" #include "mm_init.h" -#include "sparse.h" #include <linux/page-isolation.h> #define HUGE_BOOTMEM_ZONES_VALID BIT(0) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 76765c97ff68..0057fa2a16a8 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -15,10 +15,10 @@ #include <linux/mmdebug.h> #include <linux/pagewalk.h> #include <linux/pgalloc.h> +#include <linux/vmemmap-optimization.h> #include <asm/tlbflush.h> #include "hugetlb_vmemmap.h" -#include "sparse.h" /** * struct vmemmap_remap_walk - walk vmemmap page table diff --git a/mm/sparse.h b/mm/sparse.h index 326ad43bb5c3..a5111087ee3a 100644 --- a/mm/sparse.h +++ b/mm/sparse.h @@ -9,80 +9,7 @@ #define __MM_SPARSE_H #include <linux/mmzone.h> - -#ifdef CONFIG_VMEMMAP_OPTIMIZATION -static inline unsigned int section_compound_order(const struct mem_section *section) -{ - return section->compound_page_order; -} - -static inline void section_set_compound_order(struct mem_section *section, - unsigned int order) -{ - VM_WARN_ON(section_compound_order(section) && order && - section_compound_order(section) != order); - section->compound_page_order = order; -} - -static inline void section_set_compound_order_range(unsigned long pfn, - unsigned long nr_pages, unsigned int order) -{ - unsigned long section_nr = pfn_to_section_nr(pfn); - - if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) - return; - - for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) - section_set_compound_order(__nr_to_section(section_nr + i), order); -} - -static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) -{ - return section_compound_order(__pfn_to_section(pfn)); -} -#else -static inline unsigned int section_compound_order(const struct mem_section *section) -{ - return 0; -} - -static inline void section_set_compound_order(struct mem_section *section, - unsigned int order) -{ -} - -static inline void section_set_compound_order_range(unsigned long pfn, - unsigned long nr_pages, unsigned int order) -{ -} - -static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) -{ - return 0; -} -#endif - -static inline bool vmemmap_optimizable_pfn(unsigned long pfn) -{ - const unsigned int order = pfn_to_section_compound_order(pfn); - const unsigned long nr_pages = 1UL << order; - - if (!is_power_of_2(sizeof(struct page))) - return false; - - return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; -} - -static inline bool vmemmap_optimizable_order(unsigned int order) -{ - if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) - return false; - - if (!is_power_of_2(sizeof(struct page))) - return false; - - return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; -} +#include <linux/vmemmap-optimization.h> /* * mm/sparse.c @@ -142,9 +69,6 @@ static inline void sparse_sections_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP -#ifdef CONFIG_VMEMMAP_OPTIMIZATION -struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); -#endif void sparse_init_subsection_map(void); int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); -- 2.54.0
