Module Name: src
Committed By: yamt
Date: Wed Nov 2 21:55:39 UTC 2011
Added Files:
src/sys/uvm [yamt-pagecache]: uvm_page_array.c uvm_page_array.h
uvm_page_status.c
Log Message:
page cache related changes
- maintain object pages in radix tree rather than rb tree.
- reduce unnecessary page scan in putpages. esp. when an object has a ton of
pages cached but only a few of them are dirty.
- reduce the number of pmap operations by tracking page dirtiness more
precisely in uvm layer.
- fix nfs commit range tracking.
- fix nfs write clustering. XXX hack
To generate a diff of this commit:
cvs rdiff -u -r0 -r1.1.2.1 src/sys/uvm/uvm_page_array.c \
src/sys/uvm/uvm_page_array.h src/sys/uvm/uvm_page_status.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Added files:
Index: src/sys/uvm/uvm_page_array.c
diff -u /dev/null src/sys/uvm/uvm_page_array.c:1.1.2.1
--- /dev/null Wed Nov 2 21:55:39 2011
+++ src/sys/uvm/uvm_page_array.c Wed Nov 2 21:55:39 2011
@@ -0,0 +1,163 @@
+/* $NetBSD: uvm_page_array.c,v 1.1.2.1 2011/11/02 21:55:39 yamt Exp $ */
+
+/*-
+ * Copyright (c)2011 YAMAMOTO Takashi,
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.1.2.1 2011/11/02 21:55:39 yamt Exp $");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <uvm/uvm_extern.h>
+#include <uvm/uvm_object.h>
+#include <uvm/uvm_page.h>
+#include <uvm/uvm_page_array.h>
+
+/*
+ * uvm_page_array_init: initialize the array.
+ */
+
+void
+uvm_page_array_init(struct uvm_page_array *ar)
+{
+
+ ar->ar_idx = ar->ar_npages = 0;
+}
+
+/*
+ * uvm_page_array_fini: clean up the array.
+ */
+
+void
+uvm_page_array_fini(struct uvm_page_array *ar)
+{
+
+ /*
+ * currently nothing to do.
+ */
+#if defined(DIAGNOSTIC)
+ /*
+ * poison to trigger assertion in uvm_page_array_peek to
+ * detect usage errors.
+ */
+ ar->ar_npages = 1;
+ ar->ar_idx = 1000;
+#endif /* defined(DIAGNOSTIC) */
+}
+
+/*
+ * uvm_page_array_clear: forget the cached pages and initialize the array.
+ */
+
+void
+uvm_page_array_clear(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ uvm_page_array_init(ar);
+}
+
+/*
+ * uvm_page_array_peek: return the next cached page.
+ */
+
+struct vm_page *
+uvm_page_array_peek(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ if (ar->ar_idx == ar->ar_npages) {
+ return NULL;
+ }
+ return ar->ar_pages[ar->ar_idx];
+}
+
+/*
+ * uvm_page_array_advance: advance the array to the next cached page
+ */
+
+void
+uvm_page_array_advance(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ ar->ar_idx++;
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+}
+
+/*
+ * uvm_page_array_fill: lookup pages and keep them cached.
+ *
+ * return 0 on success. in that case, cache the result in the array
+ * so that they will be picked by later uvm_page_array_peek.
+ *
+ * return ENOENT if no pages are found.
+ *
+ * called with object lock held.
+ */
+
+int
+uvm_page_array_fill(struct uvm_page_array *ar, struct uvm_object *uobj,
+ voff_t off, bool dirtyonly)
+{
+ unsigned int npages;
+#if defined(DEBUG)
+ unsigned int i;
+#endif /* defined(DEBUG) */
+ const unsigned int maxpages = __arraycount(ar->ar_pages);
+
+ KASSERT(mutex_owned(uobj->vmobjlock));
+ KASSERT(uvm_page_array_peek(ar) == NULL);
+ if (dirtyonly) {
+ npages = radix_tree_gang_lookup_tagged_node(
+ &uobj->uo_pages, off >> PAGE_SHIFT,
+ (void **)ar->ar_pages, maxpages,
+ UVM_PAGE_DIRTY_TAG);
+ } else {
+ npages = radix_tree_gang_lookup_node(
+ &uobj->uo_pages, off >> PAGE_SHIFT,
+ (void **)ar->ar_pages, maxpages);
+ }
+ if (npages == 0) {
+ uvm_page_array_clear(ar);
+ return ENOENT;
+ }
+ KASSERT(npages <= maxpages);
+ ar->ar_npages = npages;
+ ar->ar_idx = 0;
+#if defined(DEBUG)
+ for (i = 0; i < ar->ar_npages; i++) {
+ struct vm_page * const pg = ar->ar_pages[i];
+
+ KASSERT(pg != NULL);
+ KASSERT(pg->uobject == uobj);
+ KASSERT(pg->offset >= off);
+ KASSERT(i == 0 || pg->offset > ar->ar_pages[i - 1]->offset);
+ }
+#endif /* defined(DEBUG) */
+ return 0;
+}
Index: src/sys/uvm/uvm_page_array.h
diff -u /dev/null src/sys/uvm/uvm_page_array.h:1.1.2.1
--- /dev/null Wed Nov 2 21:55:39 2011
+++ src/sys/uvm/uvm_page_array.h Wed Nov 2 21:55:39 2011
@@ -0,0 +1,68 @@
+/* $NetBSD: uvm_page_array.h,v 1.1.2.1 2011/11/02 21:55:39 yamt Exp $ */
+
+/*-
+ * Copyright (c)2011 YAMAMOTO Takashi,
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if !defined(_UVM_UVM_PAGE_ARRAY_H_)
+#define _UVM_UVM_PAGE_ARRAY_H_
+
+/*
+ * uvm_page_array: an array of pages.
+ *
+ * these structure and functions simply manipulate struct vm_page pointers.
+ * it's caller's responsibity to acquire and keep the object lock so that
+ * the result is valid.
+ *
+ * typical usage:
+ *
+ * struct uvm_page_array ar;
+ *
+ * uvm_page_array_init(&ar);
+ * for (uvm_page_array_fill(&ar, ...), pg = uvm_page_array_peek(&ar);
+ * pg != NULL;
+ * uvm_page_array_advance(&ar), pg = uvm_page_array_peek(&ar)) {
+ * do_something(pg);
+ * }
+ * uvm_page_array_fini(&it);
+ */
+
+struct vm_page;
+
+struct uvm_page_array {
+ struct vm_page *ar_pages[16]; /* XXX tune */
+ unsigned int ar_npages; /* valid elements in ar_pages */
+ unsigned int ar_idx; /* index in ar_pages */
+};
+
+void uvm_page_array_init(struct uvm_page_array *);
+void uvm_page_array_fini(struct uvm_page_array *);
+void uvm_page_array_clear(struct uvm_page_array *);
+struct vm_page *uvm_page_array_peek(struct uvm_page_array *);
+void uvm_page_array_advance(struct uvm_page_array *);
+int uvm_page_array_fill(struct uvm_page_array *, struct uvm_object *,
+ voff_t, bool);
+
+#endif /* defined(_UVM_UVM_ARRAY_H_) */
Index: src/sys/uvm/uvm_page_status.c
diff -u /dev/null src/sys/uvm/uvm_page_status.c:1.1.2.1
--- /dev/null Wed Nov 2 21:55:39 2011
+++ src/sys/uvm/uvm_page_status.c Wed Nov 2 21:55:39 2011
@@ -0,0 +1,155 @@
+/* $NetBSD: uvm_page_status.c,v 1.1.2.1 2011/11/02 21:55:39 yamt Exp $ */
+
+/*-
+ * Copyright (c)2011 YAMAMOTO Takashi,
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.1 2011/11/02 21:55:39 yamt Exp $");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <uvm/uvm.h>
+
+/*
+ * page dirtiness status tracking
+ *
+ * separated from uvm_page.c mainly for rump
+ */
+
+/*
+ * uvm_pagegetdirty: return the dirtiness status (one of UVM_PAGE_STATUS_
+ * values) of the page.
+ */
+
+unsigned int
+uvm_pagegetdirty(struct vm_page *pg)
+{
+ struct uvm_object * const uobj __unused = pg->uobject;
+ const uint64_t idx __unused = pg->offset >> PAGE_SHIFT;
+
+ KASSERT((~pg->flags & (PG_CLEAN|PG_DIRTY)) != 0);
+ KASSERT(uvm_page_locked_p(pg));
+ KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
+ radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+ return pg->flags & (PG_CLEAN|PG_DIRTY);
+}
+
+/*
+ * uvm_pagemarkdirty: set the dirtiness status (one of UVM_PAGE_STATUS_ values)
+ * of the page.
+ */
+
+void
+uvm_pagemarkdirty(struct vm_page *pg, unsigned int status)
+{
+ struct uvm_object * const uobj = pg->uobject;
+ const uint64_t idx = pg->offset >> PAGE_SHIFT;
+
+ KASSERT((~status & (PG_CLEAN|PG_DIRTY)) != 0);
+ KASSERT((status & ~(PG_CLEAN|PG_DIRTY)) == 0);
+ KASSERT(uvm_page_locked_p(pg));
+ KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
+ radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+
+ if (uvm_pagegetdirty(pg) == status) {
+ return;
+ }
+ /*
+ * set UVM_PAGE_DIRTY_TAG tag unless known CLEAN so that putpages can
+ * find possibly-dirty pages quickly.
+ */
+ if (uobj != NULL) {
+ if (status == UVM_PAGE_STATUS_CLEAN) {
+ radix_tree_clear_tag(&uobj->uo_pages, idx,
+ UVM_PAGE_DIRTY_TAG);
+ } else {
+ radix_tree_set_tag(&uobj->uo_pages, idx,
+ UVM_PAGE_DIRTY_TAG);
+ }
+ }
+ if (status == UVM_PAGE_STATUS_UNKNOWN) {
+ /*
+ * start relying on pmap-level dirtiness tracking.
+ */
+ pmap_clear_modify(pg);
+ }
+ pg->flags &= ~(PG_CLEAN|PG_DIRTY);
+ pg->flags |= status;
+ KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
+ radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
+}
+
+/*
+ * uvm_pagecheckdirty: check if page is dirty, and remove its dirty-marks.
+ *
+ * called with the owner locked.
+ */
+
+bool
+uvm_pagecheckdirty(struct vm_page *pg, bool protected)
+{
+ const unsigned int oldstatus = uvm_pagegetdirty(pg);
+ bool modified;
+
+ KASSERT(uvm_page_locked_p(pg));
+
+ /*
+ * if protected is true, mark the page CLEAN.
+ * otherwise mark the page UNKNOWN unless it's CLEAN.
+ *
+ * possible transitions:
+ *
+ * CLEAN -> CLEAN , modified = false
+ * UNKNOWN -> UNKNOWN, modified = true
+ * UNKNOWN -> UNKNOWN, modified = false
+ * UNKNOWN -> CLEAN , modified = true
+ * UNKNOWN -> CLEAN , modified = false
+ * DIRTY -> UNKNOWN, modified = true
+ * DIRTY -> CLEAN , modified = true
+ *
+ * pmap_clear_modify is necessary if either of
+ * oldstatus or newstatus is UVM_PAGE_STATUS_UNKNOWN.
+ */
+
+ if (oldstatus == UVM_PAGE_STATUS_CLEAN) {
+ modified = false;
+ } else {
+ const unsigned int newstatus =
+ protected ? UVM_PAGE_STATUS_CLEAN : UVM_PAGE_STATUS_UNKNOWN;
+
+ if (oldstatus == UVM_PAGE_STATUS_DIRTY) {
+ modified = true;
+ if (newstatus == UVM_PAGE_STATUS_UNKNOWN) {
+ pmap_clear_modify(pg);
+ }
+ } else {
+ modified = pmap_clear_modify(pg);
+ }
+ uvm_pagemarkdirty(pg, newstatus);
+ }
+ return modified;
+}
