Marking PUDs that map a "normal" refcounted folios as special is
against our rules documented for vm_normal_page().
Fortunately, there are not that many pud_special() check that can be
mislead and are right now rather harmless: e.g., none so far
bases decisions whether to grab a folio reference on that decision.
Well, and GUP-fast will fallback to GUP-slow. All in all, so far no big
implications as it seems.
Getting this right will get more important as we introduce
folio_normal_page_pud() and start using it in more place where we
currently special-case based on other VMA flags.
Fix it by just inlining the relevant code, making the whole
pud_none() handling cleaner.
Add folio_mk_pud() to mimic what we do with folio_mk_pmd().
While at it, make sure that the pud that is non-none is actually present
before comparing PFNs.
Fixes: dbe54153296d ("mm/huge_memory: add vmf_insert_folio_pud()")
Signed-off-by: David Hildenbrand <[email protected]>
---
include/linux/mm.h | 15 +++++++++++++++
mm/huge_memory.c | 33 +++++++++++++++++++++++----------
2 files changed, 38 insertions(+), 10 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0ef2ba0c667af..047c8261d4002 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1816,6 +1816,21 @@ static inline pmd_t folio_mk_pmd(struct folio *folio,
pgprot_t pgprot)
{
return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot));
}
+
+/**
+ * folio_mk_pud - Create a PUD for this folio
+ * @folio: The folio to create a PUD for
+ * @pgprot: The page protection bits to use
+ *
+ * Create a page table entry for the first page of this folio.
+ * This is suitable for passing to set_pud_at().
+ *
+ * Return: A page table entry suitable for mapping this folio.
+ */
+static inline pud_t folio_mk_pud(struct folio *folio, pgprot_t pgprot)
+{
+ return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot));
+}
#endif
#endif /* CONFIG_MMU */
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index f9e23dfea76f8..7b66a23089381 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1629,6 +1629,7 @@ vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf,
struct folio *folio,
pud_t *pud = vmf->pud;
struct mm_struct *mm = vma->vm_mm;
spinlock_t *ptl;
+ pud_t entry;
if (addr < vma->vm_start || addr >= vma->vm_end)
return VM_FAULT_SIGBUS;
@@ -1637,20 +1638,32 @@ vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf,
struct folio *folio,
return VM_FAULT_SIGBUS;
ptl = pud_lock(mm, pud);
-
- /*
- * If there is already an entry present we assume the folio is
- * already mapped, hence no need to take another reference. We
- * still call insert_pfn_pud() though in case the mapping needs
- * upgrading to writeable.
- */
- if (pud_none(*vmf->pud)) {
+ if (pud_none(*pud)) {
folio_get(folio);
folio_add_file_rmap_pud(folio, &folio->page, vma);
add_mm_counter(mm, mm_counter_file(folio), HPAGE_PUD_NR);
+
+ entry = folio_mk_pud(folio, vma->vm_page_prot);
+ if (write) {
+ entry = pud_mkyoung(pud_mkdirty(entry));
+ entry = maybe_pud_mkwrite(entry, vma);
+ }
+ set_pud_at(mm, addr, pud, entry);
+ update_mmu_cache_pud(vma, addr, pud);
+ } else if (pud_present(*pud) && write) {
+ /*
+ * We only allow for upgrading write permissions if the
+ * same folio is already mapped.
+ */
+ if (pud_pfn(*pud) == folio_pfn(folio)) {
+ entry = pud_mkyoung(*pud);
+ entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
+ if (pudp_set_access_flags(vma, addr, pud, entry, 1))
+ update_mmu_cache_pud(vma, addr, pud);
+ } else {
+ WARN_ON_ONCE(1);
+ }
}
- insert_pfn_pud(vma, addr, vmf->pud, pfn_to_pfn_t(folio_pfn(folio)),
- write);
spin_unlock(ptl);
return VM_FAULT_NOPAGE;
--
2.49.0
