git: 4113ea3403fc - main - vm_object: speed up page collect flush

[Doug Moore]

The branch main has been updated by dougm:

URL: 
https://cgit.FreeBSD.org/src/commit/?id=4113ea3403fc6abc5e3c51eb027ccef714c89270

commit 4113ea3403fc6abc5e3c51eb027ccef714c89270
Author:     Doug Moore <do...@freebsd.org>
AuthorDate: 2024-07-29 17:59:24 +0000
Commit:     Doug Moore <do...@freebsd.org>
CommitDate: 2024-07-29 18:02:13 +0000

    vm_object: speed up page collect flush
    
    vm_pageout_cluster prepares an array for passing to vm_pageout_flush
    by starting in the middle of a double-sized array and working out from
    the middle. Using the same technique in vm_object_page_collect_flush
    saves one loop that traverses a piece of linked list, and 80 bytes of
    amd64 binary code.
    
    Reviewed by:    kib
    Differential Revision:  https://reviews.freebsd.org/D46173
---
 sys/vm/vm_object.c | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 0af4402938ba..be21081255c2 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1115,17 +1115,14 @@ static int
 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
     int flags, boolean_t *allclean, boolean_t *eio)
 {
-       vm_page_t ma[vm_pageout_page_count], p_first, tp;
-       int count, i, mreq, runlen;
+       vm_page_t ma[2 * vm_pageout_page_count], tp;
+       int count, mreq, runlen;
 
        vm_page_lock_assert(p, MA_NOTOWNED);
        vm_page_assert_xbusied(p);
        VM_OBJECT_ASSERT_WLOCKED(object);
-
-       count = 1;
-       mreq = 0;
-
-       for (tp = p; count < vm_pageout_page_count; count++) {
+       ma[vm_pageout_page_count] = p;
+       for (count = 1, tp = p; count < vm_pageout_page_count; count++) {
                tp = vm_page_next(tp);
                if (tp == NULL || vm_page_tryxbusy(tp) == 0)
                        break;
@@ -1133,24 +1130,22 @@ vm_object_page_collect_flush(vm_object_t object, 
vm_page_t p, int pagerflags,
                        vm_page_xunbusy(tp);
                        break;
                }
+               ma[vm_pageout_page_count + count] = tp;
        }
 
-       for (p_first = p; count < vm_pageout_page_count; count++) {
-               tp = vm_page_prev(p_first);
+       for (mreq = 0, tp = p; count < vm_pageout_page_count; count++, mreq++) {
+               tp = vm_page_prev(tp);
                if (tp == NULL || vm_page_tryxbusy(tp) == 0)
                        break;
                if (!vm_object_page_remove_write(tp, flags, allclean)) {
                        vm_page_xunbusy(tp);
                        break;
                }
-               p_first = tp;
-               mreq++;
+               ma[vm_pageout_page_count - 1 - mreq] = tp;
        }
 
-       for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
-               ma[i] = tp;
-
-       vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
+       vm_pageout_flush(&ma[vm_pageout_page_count - mreq], count, pagerflags,
+           mreq, &runlen, eio);
        return (runlen);
 }