[RFC PATCH 19/37] mm: implement speculative handling in wp_page_copy()

[Michel Lespinasse]

Change wp_page_copy() to handle the speculative case.
This involves aborting speculative faults if they have to allocate an
anon_vma, and using pte_map_lock() instead of pte_offset_map_lock()
to complete the page fault.

Also change call sites to clear vmf->pte after unmapping the page table,
in order to satisfy pte_map_lock()'s preconditions.

Signed-off-by: Michel Lespinasse <mic...@lespinasse.org>
---
 mm/memory.c | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index eea72bd78d06..547d9d0ee962 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2911,20 +2911,27 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
        pte_t entry;
        int page_copied = 0;
        struct mmu_notifier_range range;
+       vm_fault_t ret = VM_FAULT_OOM;
 
-       if (unlikely(anon_vma_prepare(vma)))
-               goto oom;
+       if (unlikely(!vma->anon_vma)) {
+               if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
+                       ret = VM_FAULT_RETRY;
+                       goto out;
+               }
+               if (__anon_vma_prepare(vma))
+                       goto out;
+       }
 
        if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
                new_page = alloc_zeroed_user_highpage_movable(vma,
                                                              vmf->address);
                if (!new_page)
-                       goto oom;
+                       goto out;
        } else {
                new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
                                vmf->address);
                if (!new_page)
-                       goto oom;
+                       goto out;
 
                if (!cow_user_page(new_page, old_page, vmf)) {
                        /*
@@ -2941,7 +2948,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
        }
 
        if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
-               goto oom_free_new;
+               goto out_free_new;
        cgroup_throttle_swaprate(new_page, GFP_KERNEL);
 
        __SetPageUptodate(new_page);
@@ -2954,7 +2961,11 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
        /*
         * Re-check the pte - we dropped the lock
         */
-       vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
+       if (!pte_map_lock(vmf)) {
+               ret = VM_FAULT_RETRY;
+               /* put_page() will uncharge the page */
+               goto out_free_new;
+       }
        if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
                if (old_page) {
                        if (!PageAnon(old_page)) {
@@ -3042,12 +3053,12 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
                put_page(old_page);
        }
        return page_copied ? VM_FAULT_WRITE : 0;
-oom_free_new:
+out_free_new:
        put_page(new_page);
-oom:
+out:
        if (old_page)
                put_page(old_page);
-       return VM_FAULT_OOM;
+       return ret;
 }
 
 /**
@@ -3190,6 +3201,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
                        return wp_pfn_shared(vmf);
 
                pte_unmap_unlock(vmf->pte, vmf->ptl);
+               vmf->pte = NULL;
                return wp_page_copy(vmf);
        }
 
@@ -3228,6 +3240,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
        get_page(vmf->page);
 
        pte_unmap_unlock(vmf->pte, vmf->ptl);
+       vmf->pte = NULL;
        return wp_page_copy(vmf);
 }
 
-- 
2.20.1