The Hugepage Vmemmap Optimization (HVO) requires that struct page size is a power of two. This size is evaluated by the C compiler and currently cannot be natively evaluated by Kconfig. Therefore, the condition is_power_of_2(sizeof(struct page)) was scattered across several macros and static inline functions.
Extract the check into a preprocessor macro STRUCT_PAGE_SIZE_IS_POWER_OF_2 evaluated during the Kbuild process. Define SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED as a master toggle that is 1 only if both Kconfig CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION and the power of 2 size check are true. This allows us to completely remove all scattered sizeof(struct page) checks, making the code much cleaner and eliminating redundant logic. Additionally, mm/hugetlb_vmemmap.c and its corresponding header are now guarded by SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED. This brings an added benefit: when struct page size is not a power of 2, the compiler can entirely optimize away the unused functions in mm/hugetlb_vmemmap.c, reducing kernel image size. Signed-off-by: Muchun Song <[email protected]> --- include/linux/mm_types.h | 2 ++ include/linux/mm_types_task.h | 4 ++++ include/linux/mmzone.h | 32 +++++++++++++++----------------- include/linux/page-flags.h | 28 ++++------------------------ kernel/bounds.c | 2 ++ mm/hugetlb_vmemmap.c | 2 ++ mm/hugetlb_vmemmap.h | 4 +--- mm/internal.h | 3 --- mm/sparse.c | 6 ++---- mm/util.c | 2 +- 10 files changed, 33 insertions(+), 52 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b82..6de6c0c20f8b 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -15,7 +15,9 @@ #include <linux/cpumask.h> #include <linux/uprobes.h> #include <linux/rcupdate.h> +#ifndef __GENERATING_BOUNDS_H #include <linux/page-flags-layout.h> +#endif #include <linux/workqueue.h> #include <linux/seqlock.h> #include <linux/percpu_counter.h> diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h index 11bf319d78ec..09e5039fff97 100644 --- a/include/linux/mm_types_task.h +++ b/include/linux/mm_types_task.h @@ -17,7 +17,11 @@ #include <asm/tlbbatch.h> #endif +#ifndef __GENERATING_BOUNDS_H #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) +#else +#define ALLOC_SPLIT_PTLOCKS 0 +#endif /* * When updating this, please also update struct resident_page_types[] in diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a6900f585f9b..3a46cb0bfaaa 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -96,27 +96,26 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - /* The number of vmemmap pages required by a vmemmap-optimized folio. */ #define OPTIMIZED_FOLIO_VMEMMAP_PAGES 1 #define OPTIMIZED_FOLIO_VMEMMAP_SIZE (OPTIMIZED_FOLIO_VMEMMAP_PAGES * PAGE_SIZE) #define OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS (OPTIMIZED_FOLIO_VMEMMAP_SIZE / sizeof(struct page)) #define OPTIMIZABLE_FOLIO_MIN_ORDER (ilog2(OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS) + 1) +#if defined(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && STRUCT_PAGE_SIZE_IS_POWER_OF_2 +#define SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED 1 +/* + * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN (MAX_FOLIO_NR_PAGES * sizeof(struct page)) #define __NR_OPTIMIZABLE_FOLIO_SIZES (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1) #define NR_OPTIMIZABLE_FOLIO_SIZES \ - ((__NR_OPTIMIZABLE_FOLIO_SIZES > 0 && \ - IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) ? __NR_OPTIMIZABLE_FOLIO_SIZES : 0) + (__NR_OPTIMIZABLE_FOLIO_SIZES > 0 ? __NR_OPTIMIZABLE_FOLIO_SIZES : 0) +#else +#define MAX_FOLIO_VMEMMAP_ALIGN 0 +#define NR_OPTIMIZABLE_FOLIO_SIZES 0 +#endif enum migratetype { MIGRATE_UNMOVABLE, @@ -2015,7 +2014,7 @@ struct mem_section { */ struct page_ext *page_ext; #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED /* * The order of compound pages in this section. Typically, the section * holds compound pages of this order; a larger compound page will span @@ -2208,7 +2207,7 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long } #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED static inline void section_set_order(struct mem_section *section, unsigned int order) { VM_BUG_ON(section->order && order && section->order != order); @@ -2267,8 +2266,7 @@ static inline void section_set_compound_range(unsigned long pfn, static inline bool section_vmemmap_optimizable(const struct mem_section *section) { - return is_power_of_2(sizeof(struct page)) && - section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER; + return section_order(section) >= OPTIMIZABLE_FOLIO_MIN_ORDER; } void sparse_init_early_section(int nid, struct page *map, unsigned long pnum, diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 12665b34586c..bea934d49750 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -198,32 +198,12 @@ enum pageflags { #ifndef __GENERATING_BOUNDS_H -/* - * For tail pages, if the size of struct page is power-of-2 ->compound_info - * encodes the mask that converts the address of the tail page address to - * the head page address. - * - * Otherwise, ->compound_info has direct pointer to head pages. - */ -static __always_inline bool compound_info_has_mask(void) -{ - /* - * The approach with mask would work in the wider set of conditions, - * but it requires validating that struct pages are naturally aligned - * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. - */ - if (!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) - return false; - - return is_power_of_2(sizeof(struct page)); -} - static __always_inline unsigned long _compound_head(const struct page *page) { unsigned long info = READ_ONCE(page->compound_info); unsigned long mask; - if (!compound_info_has_mask()) { + if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) { /* Bit 0 encodes PageTail() */ if (info & 1) return info - 1; @@ -232,8 +212,8 @@ static __always_inline unsigned long _compound_head(const struct page *page) } /* - * If compound_info_has_mask() is true the rest of the info encodes - * the mask that converts the address of the tail page to the head page. + * If HVO is enabled the rest of the info encodes the mask that converts + * the address of the tail page to the head page. * * No need to clear bit 0 in the mask as 'page' always has it clear. * @@ -257,7 +237,7 @@ static __always_inline void set_compound_head(struct page *tail, unsigned int shift; unsigned long mask; - if (!compound_info_has_mask()) { + if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) { WRITE_ONCE(tail->compound_info, (unsigned long)head | 1); return; } diff --git a/kernel/bounds.c b/kernel/bounds.c index 02b619eb6106..ff2ec3834d32 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -8,6 +8,7 @@ #define __GENERATING_BOUNDS_H #define COMPILE_OFFSETS /* Include headers that define the enum constants of interest */ +#include <linux/mm_types.h> #include <linux/page-flags.h> #include <linux/mmzone.h> #include <linux/kbuild.h> @@ -30,6 +31,7 @@ int main(void) DEFINE(LRU_GEN_WIDTH, 0); DEFINE(__LRU_REFS_WIDTH, 0); #endif + DEFINE(STRUCT_PAGE_SIZE_IS_POWER_OF_2, is_power_of_2(sizeof(struct page))); /* End of constants */ return 0; diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index d595ef759bc2..0347341be156 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -21,6 +21,7 @@ #include "hugetlb_vmemmap.h" #include "internal.h" +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED /** * struct vmemmap_remap_walk - walk vmemmap page table * @@ -693,3 +694,4 @@ static int __init hugetlb_vmemmap_init(void) return 0; } late_initcall(hugetlb_vmemmap_init); +#endif diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 0022f9c5a101..bd576ef41ee7 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -12,7 +12,7 @@ #include <linux/io.h> #include <linux/memblock.h> -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#if defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && defined(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED) int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, @@ -34,8 +34,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate { int size = hugetlb_vmemmap_size(h) - OPTIMIZED_FOLIO_VMEMMAP_SIZE; - if (!is_power_of_2(sizeof(struct page))) - return 0; return size > 0 ? size : 0; } #else diff --git a/mm/internal.h b/mm/internal.h index 02064f21bfe1..121c9076f09a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1026,9 +1026,6 @@ static inline bool vmemmap_page_optimizable(const struct page *page) unsigned long pfn = page_to_pfn(page); unsigned int order = section_order(__pfn_to_section(pfn)); - if (!is_power_of_2(sizeof(struct page))) - return false; - return (pfn & ((1L << order) - 1)) >= OPTIMIZED_FOLIO_VMEMMAP_PAGE_STRUCTS; } diff --git a/mm/sparse.c b/mm/sparse.c index 77bb0113bac5..7375f66a58d5 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -404,10 +404,8 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; - if (compound_info_has_mask()) { - VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), - MAX_FOLIO_VMEMMAP_ALIGN)); - } + VM_WARN_ON_ONCE(IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED) && + !IS_ALIGNED((unsigned long)pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN)); pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); diff --git a/mm/util.c b/mm/util.c index f063fd4de1e8..783b2081ea74 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1348,7 +1348,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page) foliop = (struct folio *)page; } else { /* See compound_head() */ - if (compound_info_has_mask()) { + if (IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLED)) { unsigned long p = (unsigned long)page; foliop = (struct folio *)(p & info); -- 2.20.1
