Module Name: src
Committed By: ad
Date: Sun Dec 1 14:40:31 UTC 2019
Modified Files:
src/sys/uvm: uvm.h uvm_anon.c uvm_aobj.c uvm_km.c uvm_loan.c
uvm_pager.c uvm_pdaemon.c uvm_swap.c
Log Message:
- Adjust uvmexp.swpgonly with atomics, and make uvm_swap_data_lock static.
- A bit more __cacheline_aligned on mutexes.
To generate a diff of this commit:
cvs rdiff -u -r1.68 -r1.69 src/sys/uvm/uvm.h
cvs rdiff -u -r1.64 -r1.65 src/sys/uvm/uvm_anon.c
cvs rdiff -u -r1.128 -r1.129 src/sys/uvm/uvm_aobj.c
cvs rdiff -u -r1.147 -r1.148 src/sys/uvm/uvm_km.c
cvs rdiff -u -r1.87 -r1.88 src/sys/uvm/uvm_loan.c
cvs rdiff -u -r1.111 -r1.112 src/sys/uvm/uvm_pager.c \
src/sys/uvm/uvm_pdaemon.c
cvs rdiff -u -r1.181 -r1.182 src/sys/uvm/uvm_swap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/uvm.h
diff -u src/sys/uvm/uvm.h:1.68 src/sys/uvm/uvm.h:1.69
--- src/sys/uvm/uvm.h:1.68 Mon Jan 2 20:08:32 2017
+++ src/sys/uvm/uvm.h Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm.h,v 1.68 2017/01/02 20:08:32 cherry Exp $ */
+/* $NetBSD: uvm.h,v 1.69 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -126,7 +126,6 @@ extern struct uvm_object *uvm_kernel_obj
extern kmutex_t uvm_pageqlock; /* lock for active/inactive page q */
extern kmutex_t uvm_fpageqlock; /* lock for free page q */
extern kmutex_t uvm_kentry_lock;
-extern kmutex_t uvm_swap_data_lock;
#endif /* _KERNEL */
Index: src/sys/uvm/uvm_anon.c
diff -u src/sys/uvm/uvm_anon.c:1.64 src/sys/uvm/uvm_anon.c:1.65
--- src/sys/uvm/uvm_anon.c:1.64 Sat Oct 28 00:37:13 2017
+++ src/sys/uvm/uvm_anon.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.c,v 1.64 2017/10/28 00:37:13 pgoyette Exp $ */
+/* $NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.64 2017/10/28 00:37:13 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $");
#include "opt_uvmhist.h"
@@ -172,10 +172,8 @@ uvm_anon_dispose(struct vm_anon *anon)
#if defined(VMSWAP)
if (pg == NULL && anon->an_swslot > 0) {
/* This page is no longer only in swap. */
- mutex_enter(&uvm_swap_data_lock);
KASSERT(uvmexp.swpgonly > 0);
- uvmexp.swpgonly--;
- mutex_exit(&uvm_swap_data_lock);
+ atomic_dec_uint(&uvmexp.swpgonly);
}
#endif
Index: src/sys/uvm/uvm_aobj.c
diff -u src/sys/uvm/uvm_aobj.c:1.128 src/sys/uvm/uvm_aobj.c:1.129
--- src/sys/uvm/uvm_aobj.c:1.128 Sun Jul 28 05:28:53 2019
+++ src/sys/uvm/uvm_aobj.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.128 2019/07/28 05:28:53 msaitoh Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.129 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.128 2019/07/28 05:28:53 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.129 2019/12/01 14:40:31 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -1491,10 +1491,8 @@ uao_dropswap_range(struct uvm_object *uo
*/
if (swpgonlydelta > 0) {
- mutex_enter(&uvm_swap_data_lock);
KASSERT(uvmexp.swpgonly >= swpgonlydelta);
- uvmexp.swpgonly -= swpgonlydelta;
- mutex_exit(&uvm_swap_data_lock);
+ atomic_add_int(&uvmexp.swpgonly, -swpgonlydelta);
}
}
Index: src/sys/uvm/uvm_km.c
diff -u src/sys/uvm/uvm_km.c:1.147 src/sys/uvm/uvm_km.c:1.148
--- src/sys/uvm/uvm_km.c:1.147 Thu Nov 14 16:23:53 2019
+++ src/sys/uvm/uvm_km.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_km.c,v 1.147 2019/11/14 16:23:53 maxv Exp $ */
+/* $NetBSD: uvm_km.c,v 1.148 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -152,7 +152,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.147 2019/11/14 16:23:53 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.148 2019/12/01 14:40:31 ad Exp $");
#include "opt_uvmhist.h"
@@ -481,10 +481,7 @@ uvm_km_pgremove(vaddr_t startva, vaddr_t
mutex_exit(uobj->vmobjlock);
if (swpgonlydelta > 0) {
- mutex_enter(&uvm_swap_data_lock);
- KASSERT(uvmexp.swpgonly >= swpgonlydelta);
- uvmexp.swpgonly -= swpgonlydelta;
- mutex_exit(&uvm_swap_data_lock);
+ atomic_add_int(&uvmexp.swpgonly, -swpgonlydelta);
}
}
Index: src/sys/uvm/uvm_loan.c
diff -u src/sys/uvm/uvm_loan.c:1.87 src/sys/uvm/uvm_loan.c:1.88
--- src/sys/uvm/uvm_loan.c:1.87 Fri May 25 20:11:03 2018
+++ src/sys/uvm/uvm_loan.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_loan.c,v 1.87 2018/05/25 20:11:03 jdolecek Exp $ */
+/* $NetBSD: uvm_loan.c,v 1.88 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.87 2018/05/25 20:11:03 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.88 2019/12/01 14:40:31 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -837,7 +837,7 @@ fail:
*/
static struct uvm_object uvm_loanzero_object;
-static kmutex_t uvm_loanzero_lock;
+static kmutex_t uvm_loanzero_lock __cacheline_aligned;
static int
uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
Index: src/sys/uvm/uvm_pager.c
diff -u src/sys/uvm/uvm_pager.c:1.111 src/sys/uvm/uvm_pager.c:1.112
--- src/sys/uvm/uvm_pager.c:1.111 Sat Oct 28 00:37:13 2017
+++ src/sys/uvm/uvm_pager.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pager.c,v 1.111 2017/10/28 00:37:13 pgoyette Exp $ */
+/* $NetBSD: uvm_pager.c,v 1.112 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.111 2017/10/28 00:37:13 pgoyette Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.112 2019/12/01 14:40:31 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -77,7 +77,7 @@ const struct uvm_pagerops * const uvmpag
*/
struct vm_map *pager_map; /* XXX */
-kmutex_t pager_map_wanted_lock;
+kmutex_t pager_map_wanted_lock __cacheline_aligned;
bool pager_map_wanted; /* locked by pager map */
static vaddr_t emergva;
static int emerg_ncolors;
@@ -467,12 +467,10 @@ uvm_aio_aiodone_pages(struct vm_page **p
KASSERT(write);
/* these pages are now only in swap. */
- mutex_enter(&uvm_swap_data_lock);
if (error != ENOMEM) {
KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse);
- uvmexp.swpgonly += npages;
+ atomic_add_int(&uvmexp.swpgonly, npages);
}
- mutex_exit(&uvm_swap_data_lock);
if (error) {
if (error != ENOMEM)
uvm_swap_markbad(swslot, npages);
Index: src/sys/uvm/uvm_pdaemon.c
diff -u src/sys/uvm/uvm_pdaemon.c:1.111 src/sys/uvm/uvm_pdaemon.c:1.112
--- src/sys/uvm/uvm_pdaemon.c:1.111 Tue Oct 1 17:40:22 2019
+++ src/sys/uvm/uvm_pdaemon.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pdaemon.c,v 1.111 2019/10/01 17:40:22 chs Exp $ */
+/* $NetBSD: uvm_pdaemon.c,v 1.112 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.111 2019/10/01 17:40:22 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.112 2019/12/01 14:40:31 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -825,15 +825,12 @@ uvmpd_scan_queue(void)
} else {
slot = uao_find_swslot(uobj, pageidx);
}
- mutex_exit(slock);
-
if (slot > 0) {
/* this page is now only in swap. */
- mutex_enter(&uvm_swap_data_lock);
KASSERT(uvmexp.swpgonly < uvmexp.swpginuse);
- uvmexp.swpgonly++;
- mutex_exit(&uvm_swap_data_lock);
+ atomic_inc_uint(&uvmexp.swpgonly);
}
+ mutex_exit(slock);
continue;
}
Index: src/sys/uvm/uvm_swap.c
diff -u src/sys/uvm/uvm_swap.c:1.181 src/sys/uvm/uvm_swap.c:1.182
--- src/sys/uvm/uvm_swap.c:1.181 Sun Oct 6 05:48:00 2019
+++ src/sys/uvm/uvm_swap.c Sun Dec 1 14:40:31 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_swap.c,v 1.181 2019/10/06 05:48:00 mlelstv Exp $ */
+/* $NetBSD: uvm_swap.c,v 1.182 2019/12/01 14:40:31 ad Exp $ */
/*
* Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.181 2019/10/06 05:48:00 mlelstv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.182 2019/12/01 14:40:31 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_compat_netbsd.h"
@@ -190,6 +190,7 @@ LIST_HEAD(swap_priority, swappri);
static struct swap_priority swap_priority;
/* locks */
+static kmutex_t uvm_swap_data_lock __cacheline_aligned;
static krwlock_t swap_syscall_lock;
/* workqueue and use counter for swap to regular files */
@@ -1069,7 +1070,7 @@ uvm_swap_shutdown(struct lwp *l)
struct vnode *vp;
int error;
- printf("turning of swap...");
+ printf("turning off swap...");
rw_enter(&swap_syscall_lock, RW_WRITER);
mutex_enter(&uvm_swap_data_lock);
again:
@@ -1663,7 +1664,7 @@ uvm_swap_markbad(int startslot, int nslo
*/
KASSERT(uvmexp.swpgonly >= nslots);
- uvmexp.swpgonly -= nslots;
+ atomic_add_int(&uvmexp.swpgonly, -nslots);
sdp->swd_npgbad += nslots;
UVMHIST_LOG(pdhist, "now %jd bad", sdp->swd_npgbad, 0,0,0);
mutex_exit(&uvm_swap_data_lock);
@@ -1750,10 +1751,8 @@ uvm_swap_get(struct vm_page *page, int s
* this page is no longer only in swap.
*/
- mutex_enter(&uvm_swap_data_lock);
KASSERT(uvmexp.swpgonly > 0);
- uvmexp.swpgonly--;
- mutex_exit(&uvm_swap_data_lock);
+ atomic_dec_uint(&uvmexp.swpgonly);
}
return error;
}
