[PATCH v7 02/12] mm: apply_to_pte_range warn and fail if a large pte is encountered

[Nicholas Piggin]

apply_to_pte_range might mistake a large pte for bad, or treat it as a
page table, resulting in a crash or corruption. Add a test to warn and
return error if large entries are found.

Signed-off-by: Nicholas Piggin <npig...@gmail.com>
---
 mm/memory.c | 60 +++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index 602f4283122f..995b2e790b79 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2262,13 +2262,20 @@ static int apply_to_pmd_range(struct mm_struct *mm, 
pud_t *pud,
        }
        do {
                next = pmd_addr_end(addr, end);
-               if (create || !pmd_none_or_clear_bad(pmd)) {
-                       err = apply_to_pte_range(mm, pmd, addr, next, fn, data,
-                                                create);
-                       if (err)
-                               break;
+               if (pmd_none(*pmd) && !create)
+                       continue;
+               if (WARN_ON_ONCE(pmd_leaf(*pmd)))
+                       return -EINVAL;
+               if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
+                       if (!create)
+                               continue;
+                       pmd_clear_bad(pmd);
                }
+               err = apply_to_pte_range(mm, pmd, addr, next, fn, data, create);
+               if (err)
+                       break;
        } while (pmd++, addr = next, addr != end);
+
        return err;
 }
 
@@ -2289,13 +2296,20 @@ static int apply_to_pud_range(struct mm_struct *mm, 
p4d_t *p4d,
        }
        do {
                next = pud_addr_end(addr, end);
-               if (create || !pud_none_or_clear_bad(pud)) {
-                       err = apply_to_pmd_range(mm, pud, addr, next, fn, data,
-                                                create);
-                       if (err)
-                               break;
+               if (pud_none(*pud) && !create)
+                       continue;
+               if (WARN_ON_ONCE(pud_leaf(*pud)))
+                       return -EINVAL;
+               if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
+                       if (!create)
+                               continue;
+                       pud_clear_bad(pud);
                }
+               err = apply_to_pmd_range(mm, pud, addr, next, fn, data, create);
+               if (err)
+                       break;
        } while (pud++, addr = next, addr != end);
+
        return err;
 }
 
@@ -2316,13 +2330,20 @@ static int apply_to_p4d_range(struct mm_struct *mm, 
pgd_t *pgd,
        }
        do {
                next = p4d_addr_end(addr, end);
-               if (create || !p4d_none_or_clear_bad(p4d)) {
-                       err = apply_to_pud_range(mm, p4d, addr, next, fn, data,
-                                                create);
-                       if (err)
-                               break;
+               if (p4d_none(*p4d) && !create)
+                       continue;
+               if (WARN_ON_ONCE(p4d_leaf(*p4d)))
+                       return -EINVAL;
+               if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
+                       if (!create)
+                               continue;
+                       p4d_clear_bad(p4d);
                }
+               err = apply_to_pud_range(mm, p4d, addr, next, fn, data, create);
+               if (err)
+                       break;
        } while (p4d++, addr = next, addr != end);
+
        return err;
 }
 
@@ -2341,8 +2362,15 @@ static int __apply_to_page_range(struct mm_struct *mm, 
unsigned long addr,
        pgd = pgd_offset(mm, addr);
        do {
                next = pgd_addr_end(addr, end);
-               if (!create && pgd_none_or_clear_bad(pgd))
+               if (pgd_none(*pgd) && !create)
                        continue;
+               if (WARN_ON_ONCE(pgd_leaf(*pgd)))
+                       return -EINVAL;
+               if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
+                       if (!create)
+                               continue;
+                       pgd_clear_bad(pgd);
+               }
                err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create);
                if (err)
                        break;
-- 
2.23.0