[PATCH v5 05/25] mm: gup: allow VM_FAULT_RETRY for multiple times

[Peter Xu]

This is the gup counterpart of the change that allows the VM_FAULT_RETRY
to happen for more than once.

Reviewed-by: Jerome Glisse <jgli...@redhat.com>
Signed-off-by: Peter Xu <pet...@redhat.com>
---
 mm/gup.c     | 17 +++++++++++++----
 mm/hugetlb.c |  6 ++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c
index 58d282115d9b..ac8d5b73c212 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -647,7 +647,10 @@ static int faultin_page(struct task_struct *tsk, struct 
vm_area_struct *vma,
        if (*flags & FOLL_NOWAIT)
                fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
        if (*flags & FOLL_TRIED) {
-               VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY);
+               /*
+                * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
+                * can co-exist
+                */
                fault_flags |= FAULT_FLAG_TRIED;
        }
 
@@ -1062,17 +1065,23 @@ static __always_inline long 
__get_user_pages_locked(struct task_struct *tsk,
                if (likely(pages))
                        pages += ret;
                start += ret << PAGE_SHIFT;
+               lock_dropped = true;
 
+retry:
                /*
                 * Repeat on the address that fired VM_FAULT_RETRY
-                * without FAULT_FLAG_ALLOW_RETRY but with
+                * with both FAULT_FLAG_ALLOW_RETRY and
                 * FAULT_FLAG_TRIED.
                 */
                *locked = 1;
-               lock_dropped = true;
                down_read(&mm->mmap_sem);
                ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED,
-                                      pages, NULL, NULL);
+                                      pages, NULL, locked);
+               if (!*locked) {
+                       /* Continue to retry until we succeeded */
+                       BUG_ON(ret != 0);
+                       goto retry;
+               }
                if (ret != 1) {
                        BUG_ON(ret > 1);
                        if (!pages_done)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ba179c2fa8fb..d9c739f9a28e 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4317,8 +4317,10 @@ long follow_hugetlb_page(struct mm_struct *mm, struct 
vm_area_struct *vma,
                                fault_flags |= FAULT_FLAG_ALLOW_RETRY |
                                        FAULT_FLAG_RETRY_NOWAIT;
                        if (flags & FOLL_TRIED) {
-                               VM_WARN_ON_ONCE(fault_flags &
-                                               FAULT_FLAG_ALLOW_RETRY);
+                               /*
+                                * Note: FAULT_FLAG_ALLOW_RETRY and
+                                * FAULT_FLAG_TRIED can co-exist
+                                */
                                fault_flags |= FAULT_FLAG_TRIED;
                        }
                        ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
-- 
2.21.0