On x86-64, vmemmap_populate() normally calls sync_global_pgds() to
keep the page tables in sync; however, when DAX HVO is enabled,
vmemmap_populate_compound_pages() skips this architecture-specific
step, so omitting the sync on x86-64 can later trigger vmemmap-access
faults.
Fix this by delegating the HVO DAX decision to the architecture:
- Architectures that do not use the generic vmemmap_populate_basepages()
or vmemmap_populate_hugepages() paths (e.g. powerpc) can implement
HVO DAX directly in their own vmemmap_populate().
- Architectures that rely on the generic helpers implicitly inherit
the correct operation logic and therefore enable HVO DAX safely
without extra work in generic vmemmap_populate_basepages() or
vmemmap_populate_hugepages().
This prevents the x86-64 sync issue.
Fixes: 4917f55b4ef9 ("mm/sparse-vmemmap: improve memory savings for compound
devmaps")
Signed-off-by: Muchun Song <[email protected]>
---
arch/powerpc/include/asm/book3s/64/radix.h | 6 ------
arch/powerpc/mm/book3s64/radix_pgtable.c | 15 +++++++++-----
mm/sparse-vmemmap.c | 24 +++++++++++-----------
3 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/radix.h
b/arch/powerpc/include/asm/book3s/64/radix.h
index bde07c6f900f..2600defa2dc2 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -357,11 +357,5 @@ int radix__remove_section_mapping(unsigned long start,
unsigned long end);
#define vmemmap_can_optimize vmemmap_can_optimize
bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap
*pgmap);
#endif
-
-#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages
-int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
- unsigned long end, int node,
- struct dev_pagemap *pgmap);
#endif /* __ASSEMBLER__ */
#endif
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c
b/arch/powerpc/mm/book3s64/radix_pgtable.c
index 568500343e5f..dfa2f7dc7e15 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -1109,7 +1109,10 @@ static inline pte_t *vmemmap_pte_alloc(pmd_t *pmdp, int
node,
return pte_offset_kernel(pmdp, address);
}
-
+static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
+ unsigned long start,
+ unsigned long end, int node,
+ struct dev_pagemap *pgmap);
int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end,
int node,
struct vmem_altmap *altmap, struct
dev_pagemap *pgmap)
@@ -1122,6 +1125,8 @@ int __meminit radix__vmemmap_populate(unsigned long
start, unsigned long end, in
pmd_t *pmd;
pte_t *pte;
+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(page_to_pfn((struct page
*)start), start, end, node, pgmap);
/*
* If altmap is present, Make sure we align the start vmemmap addr
* to PAGE_SIZE so that we calculate the correct start_pfn in
@@ -1303,10 +1308,10 @@ static pte_t * __meminit
vmemmap_compound_tail_page(unsigned long addr,
return pte;
}
-int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
- unsigned long end, int node,
- struct dev_pagemap *pgmap)
+static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
+ unsigned long start,
+ unsigned long end, int
node,
+ struct dev_pagemap *pgmap)
{
/*
* we want to map things as base page size mapping so that
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 387337bba05e..d3096de04cc6 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -296,10 +296,16 @@ static int __meminit vmemmap_populate_range(unsigned long
start,
return 0;
}
+static int __meminit vmemmap_populate_compound_pages(unsigned long start,
+ unsigned long end, int
node,
+ struct dev_pagemap *pgmap);
+
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long
end,
int node, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(start, end, node, pgmap);
return vmemmap_populate_range(start, end, node, altmap, -1, 0);
}
@@ -411,6 +417,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long
start, unsigned long end,
pud_t *pud;
pmd_t *pmd;
+ if (vmemmap_can_optimize(altmap, pgmap))
+ return vmemmap_populate_compound_pages(start, end, node, pgmap);
+
for (addr = start; addr < end; addr = next) {
next = pmd_addr_end(addr, end);
@@ -453,7 +462,6 @@ int __meminit vmemmap_populate_hugepages(unsigned long
start, unsigned long end,
return 0;
}
-#ifndef vmemmap_populate_compound_pages
/*
* For compound pages bigger than section size (e.g. x86 1G compound
* pages with 2M subsection size) fill the rest of sections as tail
@@ -491,14 +499,14 @@ static pte_t * __meminit
compound_section_tail_page(unsigned long addr)
return pte;
}
-static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
- unsigned long start,
+static int __meminit vmemmap_populate_compound_pages(unsigned long start,
unsigned long end, int
node,
struct dev_pagemap *pgmap)
{
unsigned long size, addr;
pte_t *pte;
int rc;
+ unsigned long start_pfn = page_to_pfn((struct page *)start);
if (reuse_compound_section(start_pfn, pgmap)) {
pte = compound_section_tail_page(start);
@@ -544,26 +552,18 @@ static int __meminit
vmemmap_populate_compound_pages(unsigned long start_pfn,
return 0;
}
-#endif
-
struct page * __meminit __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page);
- int r;
if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
return NULL;
- if (vmemmap_can_optimize(altmap, pgmap))
- r = vmemmap_populate_compound_pages(pfn, start, end, nid,
pgmap);
- else
- r = vmemmap_populate(start, end, nid, altmap, pgmap);
-
- if (r < 0)
+ if (vmemmap_populate(start, end, nid, altmap, pgmap))
return NULL;
return pfn_to_page(pfn);
--
2.20.1
