Module Name: src
Committed By: yamt
Date: Wed Aug 1 22:34:15 UTC 2012
Modified Files:
src/sys/miscfs/genfs [yamt-pagecache]: genfs_io.c
src/sys/uvm [yamt-pagecache]: uvm_object.h uvm_page_array.c
uvm_page_array.h uvm_page_status.c uvm_pager.c uvm_vnode.c
Log Message:
- fix integrity sync.
putpages for integrity sync (fsync, msync with MS_SYNC, etc) should not
skip pages being written back by other threads.
- adapt to radix tree tag api changes.
To generate a diff of this commit:
cvs rdiff -u -r1.53.2.15 -r1.53.2.16 src/sys/miscfs/genfs/genfs_io.c
cvs rdiff -u -r1.31.2.3 -r1.31.2.4 src/sys/uvm/uvm_object.h
cvs rdiff -u -r1.1.2.5 -r1.1.2.6 src/sys/uvm/uvm_page_array.c \
src/sys/uvm/uvm_page_array.h
cvs rdiff -u -r1.1.2.7 -r1.1.2.8 src/sys/uvm/uvm_page_status.c
cvs rdiff -u -r1.107.2.3 -r1.107.2.4 src/sys/uvm/uvm_pager.c
cvs rdiff -u -r1.97.2.6 -r1.97.2.7 src/sys/uvm/uvm_vnode.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/miscfs/genfs/genfs_io.c
diff -u src/sys/miscfs/genfs/genfs_io.c:1.53.2.15 src/sys/miscfs/genfs/genfs_io.c:1.53.2.16
--- src/sys/miscfs/genfs/genfs_io.c:1.53.2.15 Wed Aug 1 21:13:45 2012
+++ src/sys/miscfs/genfs/genfs_io.c Wed Aug 1 22:34:15 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.53.2.15 2012/08/01 21:13:45 yamt Exp $ */
+/* $NetBSD: genfs_io.c,v 1.53.2.16 2012/08/01 22:34:15 yamt Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.15 2012/08/01 21:13:45 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.16 2012/08/01 22:34:15 yamt Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -834,8 +834,7 @@ genfs_do_putpages(struct vnode *vp, off_
int error;
struct vm_page *pgs[maxpages], *pg;
struct uvm_page_array a;
- bool wasclean, needs_clean, yld;
- bool async = (origflags & PGO_SYNCIO) == 0;
+ bool wasclean, needs_clean;
bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
struct lwp * const l = curlwp ? curlwp : &lwp0;
int flags;
@@ -844,6 +843,8 @@ genfs_do_putpages(struct vnode *vp, off_
bool has_trans;
bool tryclean; /* try to pull off from the syncer's list */
bool onworklst;
+ const bool integrity_sync =
+ (origflags & (PGO_LAZY|PGO_SYNCIO)) == PGO_SYNCIO;
const bool dirtyonly = (origflags & (PGO_DEACTIVATE|PGO_FREE)) == 0;
UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist);
@@ -941,45 +942,59 @@ retry:
for (;;) {
bool protected;
+ /*
+ * if we are asked to sync for integrity, we should wait on
+ * pages being written back by another threads as well.
+ */
+
pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff, 0,
- dirtyonly ? UVM_PAGE_ARRAY_FILL_DIRTYONLY : 0);
+ dirtyonly ? (UVM_PAGE_ARRAY_FILL_DIRTY |
+ (integrity_sync ? UVM_PAGE_ARRAY_FILL_WRITEBACK : 0)) : 0);
if (pg == NULL) {
break;
}
- /*
- * if the current page is not interesting, move on to the next.
- */
-
KASSERT(pg->uobject == uobj);
KASSERT((pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
(pg->flags & (PG_BUSY)) != 0);
KASSERT(pg->offset >= startoff);
KASSERT(pg->offset >= nextoff);
KASSERT(!dirtyonly ||
- uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
+ uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN ||
+ radix_tree_get_tag(&uobj->uo_pages,
+ pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG));
if (pg->offset >= endoff) {
break;
}
- if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
- KASSERT((pg->flags & PG_BUSY) != 0);
- wasclean = false;
- nextoff = pg->offset + PAGE_SIZE;
- uvm_page_array_advance(&a);
+
+ /*
+ * a preempt point.
+ */
+
+ if ((l->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD)
+ != 0) {
+ nextoff = pg->offset; /* visit this page again */
+ mutex_exit(slock);
+ preempt();
+ /*
+ * as we dropped the object lock, our cached pages can
+ * be stale.
+ */
+ uvm_page_array_clear(&a);
+ mutex_enter(slock);
continue;
}
/*
- * if the current page needs to be cleaned and it's busy,
- * wait for it to become unbusy.
+ * if the current page is busy, wait for it to become unbusy.
*/
- yld = (l->l_cpu->ci_schedstate.spc_flags &
- SPCF_SHOULDYIELD) && !pagedaemon;
- if (pg->flags & PG_BUSY || yld) {
+ if ((pg->flags & PG_BUSY) != 0) {
UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0);
- if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
- UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0);
+ if ((pg->flags & (PG_RELEASED|PG_PAGEOUT)) != 0
+ && (flags & PGO_BUSYFAIL) != 0) {
+ UVMHIST_LOG(ubchist, "busyfail %p", pg,
+ 0,0,0);
error = EDEADLK;
if (busypg != NULL)
*busypg = pg;
@@ -992,15 +1007,19 @@ retry:
*/
break;
}
- nextoff = pg->offset; /* visit this page again */
- if ((pg->flags & PG_BUSY) != 0) {
- pg->flags |= PG_WANTED;
- UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
- } else {
- KASSERT(yld);
- mutex_exit(slock);
- preempt();
+ /*
+ * don't bother to wait on other's activities
+ * unless we are asked to sync for integrity.
+ */
+ if (!integrity_sync) {
+ wasclean = false;
+ nextoff = pg->offset + PAGE_SIZE;
+ uvm_page_array_advance(&a);
+ continue;
}
+ nextoff = pg->offset; /* visit this page again */
+ pg->flags |= PG_WANTED;
+ UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
/*
* as we dropped the object lock, our cached pages can
* be stale.
@@ -1160,6 +1179,15 @@ retry:
pgs[i-1]->offset + PAGE_SIZE == tpg->offset);
KASSERT(!needs_clean || uvm_pagegetdirty(pgs[i]) !=
UVM_PAGE_STATUS_DIRTY);
+ if (needs_clean) {
+ /*
+ * mark pages as WRITEBACK so that concurrent
+ * fsync can find and wait for our activities.
+ */
+ radix_tree_set_tag(&uobj->uo_pages,
+ pgs[i]->offset >> PAGE_SHIFT,
+ UVM_PAGE_WRITEBACK_TAG);
+ }
if (tpg->offset < startoff || tpg->offset >= endoff)
continue;
if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
@@ -1254,11 +1282,11 @@ skip_scan:
#endif /* !defined(DEBUG) */
/*
- * if we found or started any i/o and we're doing sync i/o,
+ * if we found or started any i/o and we're asked to sync for integrity,
* wait for all writes to finish.
*/
- if (!wasclean && !async) {
+ if (!wasclean && integrity_sync) {
while (vp->v_numoutput != 0)
cv_wait(&vp->v_cv, slock);
}
@@ -1379,6 +1407,10 @@ genfs_do_io(struct vnode *vp, off_t off,
KASSERT(bytes != 0);
if (iowrite) {
+ /*
+ * why += 2?
+ * 1 for biodone, 1 for uvm_aio_aiodone.
+ */
mutex_enter(vp->v_interlock);
vp->v_numoutput += 2;
mutex_exit(vp->v_interlock);
Index: src/sys/uvm/uvm_object.h
diff -u src/sys/uvm/uvm_object.h:1.31.2.3 src/sys/uvm/uvm_object.h:1.31.2.4
--- src/sys/uvm/uvm_object.h:1.31.2.3 Tue Apr 17 00:09:00 2012
+++ src/sys/uvm/uvm_object.h Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_object.h,v 1.31.2.3 2012/04/17 00:09:00 yamt Exp $ */
+/* $NetBSD: uvm_object.h,v 1.31.2.4 2012/08/01 22:34:14 yamt Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -60,10 +60,11 @@ struct uvm_object {
};
/*
- * uo_pages
+ * tags for uo_pages
*/
-#define UVM_PAGE_DIRTY_TAG 0 /* might be dirty (!PG_CLEAN) */
+#define UVM_PAGE_DIRTY_TAG 1 /* might be dirty (!PG_CLEAN) */
+#define UVM_PAGE_WRITEBACK_TAG 2 /* being written back */
/*
* UVM_OBJ_KERN is a 'special' uo_refs value which indicates that the
Index: src/sys/uvm/uvm_page_array.c
diff -u src/sys/uvm/uvm_page_array.c:1.1.2.5 src/sys/uvm/uvm_page_array.c:1.1.2.6
--- src/sys/uvm/uvm_page_array.c:1.1.2.5 Wed Apr 18 13:40:44 2012
+++ src/sys/uvm/uvm_page_array.c Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page_array.c,v 1.1.2.5 2012/04/18 13:40:44 yamt Exp $ */
+/* $NetBSD: uvm_page_array.c,v 1.1.2.6 2012/08/01 22:34:14 yamt Exp $ */
/*-
* Copyright (c)2011 YAMAMOTO Takashi,
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.1.2.5 2012/04/18 13:40:44 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.1.2.6 2012/08/01 22:34:14 yamt Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -142,12 +142,17 @@ uvm_page_array_fill(struct uvm_page_arra
KASSERT(mutex_owned(uobj->vmobjlock));
#endif
KASSERT(uvm_page_array_peek(ar) == NULL);
- if ((flags & UVM_PAGE_ARRAY_FILL_DIRTYONLY) != 0) {
+ if ((flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0) {
+ unsigned int tagmask = UVM_PAGE_DIRTY_TAG;
+
+ if ((flags & UVM_PAGE_ARRAY_FILL_WRITEBACK) != 0) {
+ tagmask |= UVM_PAGE_WRITEBACK_TAG;
+ }
npages =
(backward ? radix_tree_gang_lookup_tagged_node_reverse :
radix_tree_gang_lookup_tagged_node)(
&uobj->uo_pages, off >> PAGE_SHIFT, (void **)ar->ar_pages,
- maxpages, dense, UVM_PAGE_DIRTY_TAG);
+ maxpages, dense, tagmask);
} else {
npages =
(backward ? radix_tree_gang_lookup_node_reverse :
Index: src/sys/uvm/uvm_page_array.h
diff -u src/sys/uvm/uvm_page_array.h:1.1.2.5 src/sys/uvm/uvm_page_array.h:1.1.2.6
--- src/sys/uvm/uvm_page_array.h:1.1.2.5 Wed Apr 18 13:39:28 2012
+++ src/sys/uvm/uvm_page_array.h Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page_array.h,v 1.1.2.5 2012/04/18 13:39:28 yamt Exp $ */
+/* $NetBSD: uvm_page_array.h,v 1.1.2.6 2012/08/01 22:34:14 yamt Exp $ */
/*-
* Copyright (c)2011 YAMAMOTO Takashi,
@@ -71,8 +71,9 @@ struct vm_page *uvm_page_array_fill_and_
/*
* flags for uvm_page_array_fill and uvm_page_array_fill_and_peek
*/
-#define UVM_PAGE_ARRAY_FILL_DIRTYONLY 1 /* skip known-clean pages */
-#define UVM_PAGE_ARRAY_FILL_DENSE 2 /* stop on a hole */
-#define UVM_PAGE_ARRAY_FILL_BACKWARD 4 /* descend order */
+#define UVM_PAGE_ARRAY_FILL_DIRTY 1 /* dirty pages */
+#define UVM_PAGE_ARRAY_FILL_WRITEBACK 2 /* dirty or written-back */
+#define UVM_PAGE_ARRAY_FILL_DENSE 4 /* stop on a hole */
+#define UVM_PAGE_ARRAY_FILL_BACKWARD 8 /* descend order */
#endif /* defined(_UVM_UVM_ARRAY_H_) */
Index: src/sys/uvm/uvm_page_status.c
diff -u src/sys/uvm/uvm_page_status.c:1.1.2.7 src/sys/uvm/uvm_page_status.c:1.1.2.8
--- src/sys/uvm/uvm_page_status.c:1.1.2.7 Tue Jan 24 02:11:33 2012
+++ src/sys/uvm/uvm_page_status.c Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page_status.c,v 1.1.2.7 2012/01/24 02:11:33 yamt Exp $ */
+/* $NetBSD: uvm_page_status.c,v 1.1.2.8 2012/08/01 22:34:14 yamt Exp $ */
/*-
* Copyright (c)2011 YAMAMOTO Takashi,
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.7 2012/01/24 02:11:33 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.8 2012/08/01 22:34:14 yamt Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -65,7 +65,7 @@ uvm_pagegetdirty(struct vm_page *pg)
KASSERT((~pg->flags & (PG_CLEAN|PG_DIRTY)) != 0);
KASSERT(uvm_page_locked_p(pg));
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
- radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+ !!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
return pg->flags & (PG_CLEAN|PG_DIRTY);
}
@@ -104,7 +104,7 @@ uvm_pagemarkdirty(struct vm_page *pg, un
KASSERT((newstatus & ~(PG_CLEAN|PG_DIRTY)) == 0);
KASSERT(uvm_page_locked_p(pg));
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
- radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+ !!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
if (oldstatus == newstatus) {
return;
@@ -131,7 +131,7 @@ uvm_pagemarkdirty(struct vm_page *pg, un
pg->flags &= ~(PG_CLEAN|PG_DIRTY);
pg->flags |= newstatus;
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
- radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+ !!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
if ((pg->pqflags & PQ_STAT) != 0) {
stat_update((pg->pqflags & PQ_SWAPBACKED) != 0,
oldstatus, newstatus);
Index: src/sys/uvm/uvm_pager.c
diff -u src/sys/uvm/uvm_pager.c:1.107.2.3 src/sys/uvm/uvm_pager.c:1.107.2.4
--- src/sys/uvm/uvm_pager.c:1.107.2.3 Tue Apr 17 00:09:00 2012
+++ src/sys/uvm/uvm_pager.c Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pager.c,v 1.107.2.3 2012/04/17 00:09:00 yamt Exp $ */
+/* $NetBSD: uvm_pager.c,v 1.107.2.4 2012/08/01 22:34:14 yamt Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.107.2.3 2012/04/17 00:09:00 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.107.2.4 2012/08/01 22:34:14 yamt Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -368,6 +368,13 @@ uvm_aio_aiodone_pages(struct vm_page **p
}
#endif /* defined(VMSWAP) */
+ if (write && uobj != NULL) {
+ KASSERT(radix_tree_get_tag(&uobj->uo_pages,
+ pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG));
+ radix_tree_clear_tag(&uobj->uo_pages,
+ pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG);
+ }
+
/*
* process errors. for reads, just mark the page to be freed.
* for writes, if the error was ENOMEM, we assume this was
Index: src/sys/uvm/uvm_vnode.c
diff -u src/sys/uvm/uvm_vnode.c:1.97.2.6 src/sys/uvm/uvm_vnode.c:1.97.2.7
--- src/sys/uvm/uvm_vnode.c:1.97.2.6 Wed Aug 1 21:12:23 2012
+++ src/sys/uvm/uvm_vnode.c Wed Aug 1 22:34:14 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_vnode.c,v 1.97.2.6 2012/08/01 21:12:23 yamt Exp $ */
+/* $NetBSD: uvm_vnode.c,v 1.97.2.7 2012/08/01 22:34:14 yamt Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -45,7 +45,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.97.2.6 2012/08/01 21:12:23 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.97.2.7 2012/08/01 22:34:14 yamt Exp $");
#include "opt_uvmhist.h"
@@ -245,7 +245,7 @@ uvn_findpage(struct uvm_object *uobj, vo
const unsigned int fillflags =
((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) |
((flags & UFP_DIRTYONLY) ?
- (UVM_PAGE_ARRAY_FILL_DIRTYONLY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
+ (UVM_PAGE_ARRAY_FILL_DIRTY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
UVMHIST_FUNC("uvn_findpage"); UVMHIST_CALLED(ubchist);
UVMHIST_LOG(ubchist, "vp %p off 0x%lx", uobj, offset,0,0);
