Module Name: src
Committed By: rmind
Date: Sat Aug 6 17:25:04 UTC 2011
Modified Files:
src/sys/uvm: uvm_amap.c uvm_anon.c uvm_anon.h uvm_fault.c uvm_loan.c
uvm_map.c
Log Message:
- Rework uvm_anfree() into uvm_anon_freelst(), which always drops the lock.
- Free anons in uvm_anon_freelst() without lock held.
- Mechanic sync to unused loaning code.
To generate a diff of this commit:
cvs rdiff -u -r1.101 -r1.102 src/sys/uvm/uvm_amap.c
cvs rdiff -u -r1.58 -r1.59 src/sys/uvm/uvm_anon.c
cvs rdiff -u -r1.29 -r1.30 src/sys/uvm/uvm_anon.h
cvs rdiff -u -r1.189 -r1.190 src/sys/uvm/uvm_fault.c
cvs rdiff -u -r1.80 -r1.81 src/sys/uvm/uvm_loan.c
cvs rdiff -u -r1.302 -r1.303 src/sys/uvm/uvm_map.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/uvm_amap.c
diff -u src/sys/uvm/uvm_amap.c:1.101 src/sys/uvm/uvm_amap.c:1.102
--- src/sys/uvm/uvm_amap.c:1.101 Tue Jul 5 13:47:24 2011
+++ src/sys/uvm/uvm_amap.c Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_amap.c,v 1.101 2011/07/05 13:47:24 yamt Exp $ */
+/* $NetBSD: uvm_amap.c,v 1.102 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.101 2011/07/05 13:47:24 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.102 2011/08/06 17:25:03 rmind Exp $");
#include "opt_uvmhist.h"
@@ -365,9 +365,8 @@
amap_pp_adjref(amap, slotoff + slotmapped,
slotadd, 1, &tofree);
}
- uvm_anfree(tofree);
#endif
- amap_unlock(amap);
+ uvm_anon_freelst(amap, tofree);
UVMHIST_LOG(maphist,
"<- done (case 1f), amap = 0x%x, sltneed=%d",
amap, slotneed, 0, 0);
@@ -382,9 +381,8 @@
amap_pp_adjref(amap, slotoff, slotadd, 1,
&tofree);
}
- uvm_anfree(tofree);
#endif
- amap_unlock(amap);
+ uvm_anon_freelst(amap, tofree);
UVMHIST_LOG(maphist,
"<- done (case 1b), amap = 0x%x, sltneed=%d",
amap, slotneed, 0, 0);
@@ -412,8 +410,7 @@
}
#endif
amap->am_nslot = slotneed;
- uvm_anfree(tofree);
- amap_unlock(amap);
+ uvm_anon_freelst(amap, tofree);
/*
* no need to zero am_anon since that was done at
@@ -614,8 +611,8 @@
oldnslots = amap->am_maxslot;
amap->am_maxslot = slotalloc;
- uvm_anfree(tofree);
- amap_unlock(amap);
+ uvm_anon_freelst(amap, tofree);
+
kmem_free(oldsl, oldnslots * sizeof(*oldsl));
kmem_free(oldbck, oldnslots * sizeof(*oldbck));
kmem_free(oldover, oldnslots * sizeof(*oldover));
@@ -694,6 +691,7 @@
void
amap_wipeout(struct vm_amap *amap)
{
+ struct vm_anon *tofree = NULL;
u_int lcv;
UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
@@ -724,11 +722,12 @@
anon->an_ref, 0, 0);
/*
- * Drop the reference, and free the anon, if it is last.
+ * Drop the reference. Defer freeing.
*/
if (--anon->an_ref == 0) {
- uvm_anfree(anon);
+ anon->an_link = tofree;
+ tofree = anon;
}
if (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) {
preempt();
@@ -740,7 +739,7 @@
*/
amap->am_nused = 0;
- amap_unlock(amap);
+ uvm_anon_freelst(amap, tofree);
amap_free(amap);
UVMHIST_LOG(maphist,"<- done!", 0,0,0,0);
}
@@ -915,7 +914,6 @@
len >> PAGE_SHIFT, -1, &tofree);
}
#endif
- uvm_anfree(tofree);
/*
* If we referenced any anons, then share the source amap's lock.
@@ -927,9 +925,11 @@
amap->am_lock = srcamap->am_lock;
mutex_obj_hold(amap->am_lock);
}
- amap_unlock(srcamap);
- if (amap->am_lock == NULL)
+ uvm_anon_freelst(srcamap, tofree);
+
+ if (amap->am_lock == NULL) {
amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
+ }
amap_list_insert(amap);
/*
@@ -1047,9 +1047,10 @@
if (nanon) {
nanon->an_ref--;
KASSERT(nanon->an_ref == 0);
- uvm_anfree(nanon);
+ uvm_anon_freelst(amap, nanon);
+ } else {
+ amap_unlock(amap);
}
- amap_unlock(amap);
uvm_wait("cownowpage");
goto ReStart;
}
@@ -1279,7 +1280,7 @@
if (--anon->an_ref == 0) {
/*
* Eliminated the last reference to an anon - defer
- * freeing as uvm_anfree() can unlock the amap.
+ * freeing as uvm_anon_freelst() will unlock the amap.
*/
anon->an_link = *tofree;
*tofree = anon;
@@ -1543,6 +1544,8 @@
amap_adjref_anons(struct vm_amap *amap, vaddr_t offset, vsize_t len,
int refv, bool all)
{
+ struct vm_anon *tofree = NULL;
+
#ifdef UVM_AMAP_PPREF
KASSERT(mutex_owned(amap->am_lock));
@@ -1550,16 +1553,14 @@
amap_pp_establish(amap, offset);
}
if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
- struct vm_anon *tofree = NULL;
-
if (all) {
amap_pp_adjref(amap, 0, amap->am_nslot, refv, &tofree);
} else {
amap_pp_adjref(amap, offset, len, refv, &tofree);
}
- uvm_anfree(tofree);
}
#endif
+ uvm_anon_freelst(amap, tofree);
}
/*
@@ -1580,7 +1581,6 @@
}
amap->am_ref++;
amap_adjref_anons(amap, offset, len, 1, (flags & AMAP_REFALL) != 0);
- amap_unlock(amap);
UVMHIST_LOG(maphist,"<- done! amap=0x%x", amap, 0, 0, 0);
}
@@ -1620,7 +1620,6 @@
amap->am_flags &= ~AMAP_SHARED;
}
amap_adjref_anons(amap, offset, len, -1, all);
- amap_unlock(amap);
UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
}
Index: src/sys/uvm/uvm_anon.c
diff -u src/sys/uvm/uvm_anon.c:1.58 src/sys/uvm/uvm_anon.c:1.59
--- src/sys/uvm/uvm_anon.c:1.58 Tue Jul 5 13:47:24 2011
+++ src/sys/uvm/uvm_anon.c Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.c,v 1.58 2011/07/05 13:47:24 yamt Exp $ */
+/* $NetBSD: uvm_anon.c,v 1.59 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.58 2011/07/05 13:47:24 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.59 2011/08/06 17:25:03 rmind Exp $");
#include "opt_uvmhist.h"
@@ -62,6 +62,7 @@
struct vm_anon *anon = object;
anon->an_ref = 0;
+ anon->an_lock = NULL;
anon->an_page = NULL;
#if defined(VMSWAP)
anon->an_swslot = 0;
@@ -82,33 +83,32 @@
anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
if (anon) {
KASSERT(anon->an_ref == 0);
+ KASSERT(anon->an_lock == NULL);
KASSERT(anon->an_page == NULL);
#if defined(VMSWAP)
KASSERT(anon->an_swslot == 0);
#endif
anon->an_ref = 1;
- anon->an_lock = NULL;
}
return anon;
}
/*
- * uvm_anfree1: free a single anon.
+ * uvm_anon_dispose: free any resident page or swap resources of anon.
*
* => anon must be removed from the amap (if anon was in an amap).
- * => amap must be locked or anon must not be associated.
- * => amap lock may be dropped and re-acquired here.
+ * => amap must be locked; we may drop and re-acquire the lock here.
*/
static void
-uvm_anfree1(struct vm_anon *anon)
+uvm_anon_dispose(struct vm_anon *anon)
{
struct vm_page *pg = anon->an_page;
- UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist);
+ UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0);
- KASSERT(anon->an_lock == NULL || mutex_owned(anon->an_lock));
+ KASSERT(mutex_owned(anon->an_lock));
/*
* If there is a resident page and it is loaned, then anon may not
@@ -180,35 +180,57 @@
#endif
/*
- * Free any swap resources, leave a page replacement hint, drop
- * the reference on lock and finally destroy the anon itself.
+ * Free any swap resources, leave a page replacement hint.
*/
uvm_anon_dropswap(anon);
uvmpdpol_anfree(anon);
+ UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
+}
+
+/*
+ * uvm_anon_free: free a single anon.
+ *
+ * => anon must be already disposed.
+ */
+void
+uvm_anon_free(struct vm_anon *anon)
+{
+ KASSERT(anon->an_ref == 0);
+ KASSERT(anon->an_lock == NULL);
KASSERT(anon->an_page == NULL);
#if defined(VMSWAP)
KASSERT(anon->an_swslot == 0);
#endif
-
pool_cache_put(&uvm_anon_cache, anon);
- UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
}
/*
- * uvm_anfree: free a linked list of anon structures.
+ * uvm_anon_freelst: free a linked list of anon structures.
+ *
+ * => anon must be locked, we will unlock it.
*/
void
-uvm_anfree(struct vm_anon *anon)
+uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst)
{
- struct vm_anon *next;
+ struct vm_anon *anon = anonlst;
+
+ KASSERT(mutex_owned(amap->am_lock));
- for (; anon != NULL; anon = next) {
- /* Note: clearing an_link also clears a reference count. */
- next = anon->an_link;
- anon->an_link = NULL;
- uvm_anfree1(anon);
+ while (anon) {
+ uvm_anon_dispose(anon);
+ anon = anon->an_link;
+ }
+ amap_unlock(amap);
+
+ while (anonlst) {
+ anon = anonlst->an_link;
+ /* Note: clears an_ref as well. */
+ anonlst->an_link = NULL;
+ anonlst->an_lock = NULL;
+ uvm_anon_free(anonlst);
+ anonlst = anon;
}
}
@@ -403,9 +425,9 @@
uvm_anon_release(struct vm_anon *anon)
{
struct vm_page *pg = anon->an_page;
- kmutex_t *lock = anon->an_lock;
+ kmutex_t *lock;
- KASSERT(mutex_owned(lock));
+ KASSERT(mutex_owned(anon->an_lock));
KASSERT(pg != NULL);
KASSERT((pg->flags & PG_RELEASED) != 0);
KASSERT((pg->flags & PG_BUSY) != 0);
@@ -417,10 +439,13 @@
mutex_enter(&uvm_pageqlock);
uvm_pagefree(pg);
mutex_exit(&uvm_pageqlock);
+ mutex_exit(anon->an_lock);
KASSERT(anon->an_page == NULL);
- uvm_anfree(anon);
- mutex_exit(lock);
+ /* Note: extra reference is held for PG_RELEASED case. */
+ lock = anon->an_lock;
+ anon->an_lock = NULL;
+ uvm_anon_free(anon);
mutex_obj_free(lock);
}
Index: src/sys/uvm/uvm_anon.h
diff -u src/sys/uvm/uvm_anon.h:1.29 src/sys/uvm/uvm_anon.h:1.30
--- src/sys/uvm/uvm_anon.h:1.29 Fri Jun 24 01:39:22 2011
+++ src/sys/uvm/uvm_anon.h Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.h,v 1.29 2011/06/24 01:39:22 rmind Exp $ */
+/* $NetBSD: uvm_anon.h,v 1.30 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -100,7 +100,8 @@
*/
struct vm_anon *uvm_analloc(void);
-void uvm_anfree(struct vm_anon *);
+void uvm_anon_free(struct vm_anon *);
+void uvm_anon_freelst(struct vm_amap *, struct vm_anon *);
void uvm_anon_init(void);
struct vm_page *uvm_anon_lockloanpg(struct vm_anon *);
#if defined(VMSWAP)
Index: src/sys/uvm/uvm_fault.c
diff -u src/sys/uvm/uvm_fault.c:1.189 src/sys/uvm/uvm_fault.c:1.190
--- src/sys/uvm/uvm_fault.c:1.189 Tue Jul 5 13:47:24 2011
+++ src/sys/uvm/uvm_fault.c Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_fault.c,v 1.189 2011/07/05 13:47:24 yamt Exp $ */
+/* $NetBSD: uvm_fault.c,v 1.190 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.189 2011/07/05 13:47:24 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.190 2011/08/06 17:25:03 rmind Exp $");
#include "opt_uvmhist.h"
@@ -846,7 +846,7 @@
flt.anon_spare->an_ref--;
KASSERT(flt.anon_spare->an_ref == 0);
KASSERT(flt.anon_spare->an_lock == NULL);
- uvm_anfree(flt.anon_spare);
+ uvm_anon_free(flt.anon_spare);
}
return error;
}
Index: src/sys/uvm/uvm_loan.c
diff -u src/sys/uvm/uvm_loan.c:1.80 src/sys/uvm/uvm_loan.c:1.81
--- src/sys/uvm/uvm_loan.c:1.80 Sun Jun 12 03:36:03 2011
+++ src/sys/uvm/uvm_loan.c Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_loan.c,v 1.80 2011/06/12 03:36:03 rmind Exp $ */
+/* $NetBSD: uvm_loan.c,v 1.81 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.80 2011/06/12 03:36:03 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.81 2011/08/06 17:25:03 rmind Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -750,10 +750,9 @@
*/
if (pg->uanon) {
+ /* XXX: locking */
anon = pg->uanon;
- mutex_enter(anon->an_lock);
anon->an_ref++;
- mutex_exit(anon->an_lock);
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
@@ -773,23 +772,19 @@
if (anon == NULL) {
goto fail;
}
- anon->an_page = pg;
- pg->uanon = anon;
mutex_enter(&uvm_pageqlock);
if (pg->wire_count > 0) {
mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
- pg->uanon = NULL;
- anon->an_page = NULL;
- anon->an_ref--;
- uvm_anfree(anon);
goto fail;
}
if (pg->loan_count == 0) {
pmap_page_protect(pg, VM_PROT_READ);
}
pg->loan_count++;
- anon->an_lock =
+ pg->uanon = anon;
+ anon->an_page = pg;
+ anon->an_lock = /* TODO: share amap lock */
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
if (pg->flags & PG_WANTED) {
@@ -814,6 +809,10 @@
pg->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
uvmfault_unlockall(ufi, amap, uobj, NULL);
+ if (anon) {
+ anon->an_ref--;
+ uvm_anon_free(anon);
+ }
#endif /* notdef */
return (-1);
}
@@ -937,20 +936,18 @@
uvm_unloananon(struct vm_anon **aloans, int nanons)
{
#ifdef notdef
- struct vm_anon *anon;
+ struct vm_anon *anon, *to_free = NULL;
+ /* TODO: locking */
+ amap_lock(amap);
while (nanons-- > 0) {
- int refs;
-
anon = *aloans++;
- mutex_enter(anon->an_lock);
- refs = --anon->an_ref;
- mutex_exit(anon->an_lock);
-
- if (refs == 0) {
- uvm_anfree(anon);
+ if (--anon->an_ref == 0) {
+ anon->an_link = to_free;
+ to_free = anon;
}
}
+ uvm_anon_freelst(amap, to_free);
#endif /* notdef */
}
Index: src/sys/uvm/uvm_map.c
diff -u src/sys/uvm/uvm_map.c:1.302 src/sys/uvm/uvm_map.c:1.303
--- src/sys/uvm/uvm_map.c:1.302 Sat Jul 30 20:05:46 2011
+++ src/sys/uvm/uvm_map.c Sat Aug 6 17:25:03 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.302 2011/07/30 20:05:46 martin Exp $ */
+/* $NetBSD: uvm_map.c,v 1.303 2011/08/06 17:25:03 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.302 2011/07/30 20:05:46 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.303 2011/08/06 17:25:03 rmind Exp $");
#include "opt_ddb.h"
#include "opt_uvmhist.h"
@@ -3947,10 +3947,11 @@
if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
goto flush_object;
- amap_lock(amap);
- anon_tofree = NULL;
offset = start - current->start;
size = MIN(end, current->end) - start;
+ anon_tofree = NULL;
+
+ amap_lock(amap);
for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
anon = amap_lookup(¤t->aref, offset);
if (anon == NULL)
@@ -4012,8 +4013,7 @@
continue;
}
}
- uvm_anfree(anon_tofree);
- amap_unlock(amap);
+ uvm_anon_freelst(amap, anon_tofree);
flush_object:
/*
