[RFC, PATCHv1 08/28] x86/gup: add 5-level paging support

[Kirill A. Shutemov]

It's simply extension for one more page table level.

Signed-off-by: Kirill A. Shutemov <kirill.shute...@linux.intel.com>
---
 arch/x86/mm/gup.c | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index b8b6a60b32cf..8b69adc10988 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -76,9 +76,9 @@ static void undo_dev_pagemap(int *nr, int nr_start, struct 
page **pages)
 }
 
 /*
- * 'pteval' can come from a pte, pmd or pud.  We only check
+ * 'pteval' can come from a pte, pmd, pud or p4d.  We only check
  * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
- * same value on all 3 types.
+ * same value on all 4 types.
  */
 static inline int pte_allows_gup(unsigned long pteval, int write)
 {
@@ -270,13 +270,13 @@ static noinline int gup_huge_pud(pud_t pud, unsigned long 
addr,
        return 1;
 }
 
-static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
                        int write, struct page **pages, int *nr)
 {
        unsigned long next;
        pud_t *pudp;
 
-       pudp = pud_offset(&pgd, addr);
+       pudp = pud_offset(&p4d, addr);
        do {
                pud_t pud = *pudp;
 
@@ -295,6 +295,27 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, 
unsigned long end,
        return 1;
 }
 
+static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
+                       int write, struct page **pages, int *nr)
+{
+       unsigned long next;
+       p4d_t *p4dp;
+
+       p4dp = p4d_offset(&pgd, addr);
+       do {
+               p4d_t p4d = *p4dp;
+
+               next = p4d_addr_end(addr, end);
+               if (p4d_none(p4d))
+                       return 0;
+               BUILD_BUG_ON(p4d_large(p4d));
+               if (!gup_pud_range(p4d, addr, next, write, pages, nr))
+                       return 0;
+       } while (p4dp++, addr = next, addr != end);
+
+       return 1;
+}
+
 /*
  * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
  * back to the regular GUP.
@@ -343,7 +364,7 @@ int __get_user_pages_fast(unsigned long start, int 
nr_pages, int write,
                next = pgd_addr_end(addr, end);
                if (pgd_none(pgd))
                        break;
-               if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+               if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
                        break;
        } while (pgdp++, addr = next, addr != end);
        local_irq_restore(flags);
@@ -415,7 +436,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, 
int write,
                next = pgd_addr_end(addr, end);
                if (pgd_none(pgd))
                        goto slow;
-               if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+               if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
                        goto slow;
        } while (pgdp++, addr = next, addr != end);
        local_irq_enable();
-- 
2.10.2