Module Name: src
Committed By: ad
Date: Sat May 30 20:16:14 UTC 2020
Modified Files:
src/sys/kern: vfs_cache.c vfs_lookup.c
src/sys/sys: namei.src
Log Message:
A couple of small changes to lookup that cut 5-10% system time from
"build.sh release" on my test system:
- Crossing mount points during lookup is slow because the set up for, and
act of doing VFS_ROOT() is quite involved. Use the name cache to help
with this. Cache an "impossible" zero-length name with covered vnodes,
that points to the root of the file system mounted there. Use it to cross
mounts. When cache_purge() is called on either of the vnodes involved the
cache entry will disappear. All of the needed calls for that are already
in place (vnode reclaim, unmount, etc).
- In lookup_fastforward(), if the the last component has been found and the
parent directory (searchdir) is not going to be returned, then don't get a
reference to it.
To generate a diff of this commit:
cvs rdiff -u -r1.145 -r1.146 src/sys/kern/vfs_cache.c
cvs rdiff -u -r1.220 -r1.221 src/sys/kern/vfs_lookup.c
cvs rdiff -u -r1.57 -r1.58 src/sys/sys/namei.src
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.145 src/sys/kern/vfs_cache.c:1.146
--- src/sys/kern/vfs_cache.c:1.145 Sat May 30 18:06:17 2020
+++ src/sys/kern/vfs_cache.c Sat May 30 20:16:14 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.145 2020/05/30 18:06:17 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.146 2020/05/30 20:16:14 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -172,7 +172,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.145 2020/05/30 18:06:17 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.146 2020/05/30 20:16:14 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -269,6 +269,15 @@ int cache_stat_interval __read_mostly =
static struct sysctllog *cache_sysctllog;
/*
+ * This is a dummy name that cannot usually occur anywhere in the cache nor
+ * file system. It's used when caching the root vnode of mounted file
+ * systems. The name is attached to the directory that the file system is
+ * mounted on.
+ */
+static const char cache_mp_name[] = "";
+static const int cache_mp_nlen = sizeof(cache_mp_name) - 1;
+
+/*
* Red-black tree stuff.
*/
static const rb_tree_ops_t cache_rbtree_ops = {
@@ -507,6 +516,8 @@ cache_lookup(struct vnode *dvp, const ch
bool hit;
krw_t op;
+ KASSERT(namelen != cache_mp_nlen || name == cache_mp_name);
+
/* Establish default result values */
if (iswht_ret != NULL) {
*iswht_ret = 0;
@@ -630,6 +641,8 @@ cache_lookup_linked(struct vnode *dvp, c
uint64_t key;
int error;
+ KASSERT(namelen != cache_mp_nlen || name == cache_mp_name);
+
/* If disabled, or file system doesn't support this, bail out. */
if (__predict_false((dvp->v_mount->mnt_iflag & IMNT_NCLOOKUP) == 0)) {
return false;
@@ -714,6 +727,7 @@ cache_lookup_linked(struct vnode *dvp, c
}
if (ncp->nc_vp == NULL) {
/* found negative entry; vn is already null from above */
+ KASSERT(namelen != cache_mp_nlen && name != cache_mp_name);
COUNT(ncs_neghits);
} else {
COUNT(ncs_goodhits); /* XXX can be "badhits" */
@@ -797,6 +811,13 @@ cache_revlookup(struct vnode *vp, struct
nlen = ncp->nc_nlen;
/*
+ * Ignore mountpoint entries.
+ */
+ if (ncp->nc_nlen == cache_mp_nlen) {
+ continue;
+ }
+
+ /*
* The queue is partially sorted. Once we hit dots, nothing
* else remains but dots and dotdots, so bail out.
*/
@@ -866,6 +887,8 @@ cache_enter(struct vnode *dvp, struct vn
struct namecache *ncp, *oncp;
int total;
+ KASSERT(namelen != cache_mp_nlen || name == cache_mp_name);
+
/* First, check whether we can/should add a cache entry. */
if ((cnflags & MAKEENTRY) == 0 ||
__predict_false(namelen > cache_maxlen)) {
@@ -1002,6 +1025,49 @@ cache_have_id(struct vnode *vp)
}
/*
+ * Enter a mount point. cvp is the covered vnode, and rvp is the root of
+ * the mounted file system.
+ */
+void
+cache_enter_mount(struct vnode *cvp, struct vnode *rvp)
+{
+
+ KASSERT(vrefcnt(cvp) > 0);
+ KASSERT(vrefcnt(rvp) > 0);
+ KASSERT(cvp->v_type == VDIR);
+ KASSERT((rvp->v_vflag & VV_ROOT) != 0);
+
+ if (rvp->v_type == VDIR) {
+ cache_enter(cvp, rvp, cache_mp_name, cache_mp_nlen, MAKEENTRY);
+ }
+}
+
+/*
+ * Look up a cached mount point. Used in the strongly locked path.
+ */
+bool
+cache_lookup_mount(struct vnode *dvp, struct vnode **vn_ret)
+{
+ bool ret;
+
+ ret = cache_lookup(dvp, cache_mp_name, cache_mp_nlen, LOOKUP,
+ MAKEENTRY, NULL, vn_ret);
+ KASSERT((*vn_ret != NULL) == ret);
+ return ret;
+}
+
+/*
+ * Try to cross a mount point. For use with cache_lookup_linked().
+ */
+bool
+cache_cross_mount(struct vnode **dvp, krwlock_t **plock)
+{
+
+ return cache_lookup_linked(*dvp, cache_mp_name, cache_mp_nlen,
+ dvp, plock, FSCRED);
+}
+
+/*
* Name cache initialization, from vfs_init() when the system is booting.
*/
void
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.220 src/sys/kern/vfs_lookup.c:1.221
--- src/sys/kern/vfs_lookup.c:1.220 Tue May 26 18:38:37 2020
+++ src/sys/kern/vfs_lookup.c Sat May 30 20:16:14 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.220 2020/05/26 18:38:37 ad Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.221 2020/05/30 20:16:14 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.220 2020/05/26 18:38:37 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.221 2020/05/30 20:16:14 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -925,7 +925,7 @@ lookup_crossmount(struct namei_state *st
bool *searchdir_locked)
{
struct componentname *cnp = state->cnp;
- struct vnode *foundobj;
+ struct vnode *foundobj, *vp;
struct vnode *searchdir;
struct mount *mp;
int error, lktype;
@@ -954,38 +954,65 @@ lookup_crossmount(struct namei_state *st
(mp = foundobj->v_mountedhere) != NULL &&
(cnp->cn_flags & NOCROSSMOUNT) == 0) {
KASSERTMSG(searchdir != foundobj, "same vn %p", searchdir);
+
/*
- * First get the vnode stable. LK_SHARED works brilliantly
- * here because almost nothing else wants to lock the
- * covered vnode.
+ * Try the namecache first. If that doesn't work, do
+ * it the hard way.
*/
- error = vn_lock(foundobj, LK_SHARED);
- if (error != 0) {
+ if (cache_lookup_mount(foundobj, &vp)) {
vrele(foundobj);
- foundobj = NULL;
- break;
- }
+ foundobj = vp;
+ } else {
+ /* First get the vnode stable. */
+ error = vn_lock(foundobj, LK_SHARED);
+ if (error != 0) {
+ vrele(foundobj);
+ foundobj = NULL;
+ break;
+ }
- /* Then check to see if something is still mounted on it. */
- if ((mp = foundobj->v_mountedhere) == NULL) {
+ /*
+ * Check to see if something is still mounted on it.
+ */
+ if ((mp = foundobj->v_mountedhere) == NULL) {
+ VOP_UNLOCK(foundobj);
+ break;
+ }
+
+ /*
+ * Get a reference to the mountpoint, and unlock
+ * foundobj.
+ */
+ error = vfs_busy(mp);
VOP_UNLOCK(foundobj);
- break;
- }
+ if (error != 0) {
+ vrele(foundobj);
+ foundobj = NULL;
+ break;
+ }
- /* Get a reference to the mountpoint, and ditch foundobj. */
- error = vfs_busy(mp);
- vput(foundobj);
- if (error != 0) {
- foundobj = NULL;
- break;
- }
+ /*
+ * Now get a reference on the root vnode.
+ * XXX Future - maybe allow only VDIR here.
+ */
+ error = VFS_ROOT(mp, LK_NONE, &vp);
- /* Now get a reference on the root vnode, and drop mount. */
- error = VFS_ROOT(mp, LK_NONE, &foundobj);
- vfs_unbusy(mp);
- if (error) {
- foundobj = NULL;
- break;
+ /*
+ * If successful, enter it into the cache while
+ * holding the mount busy (competing with unmount).
+ */
+ if (error == 0) {
+ cache_enter_mount(foundobj, vp);
+ }
+
+ /* Finally, drop references to foundobj & mountpoint. */
+ vrele(foundobj);
+ vfs_unbusy(mp);
+ if (error) {
+ foundobj = NULL;
+ break;
+ }
+ foundobj = vp;
}
/*
@@ -1261,6 +1288,7 @@ lookup_fastforward(struct namei_state *s
int error, error2;
size_t oldpathlen;
const char *oldnameptr;
+ bool terminal;
/*
* Eat as many path name components as possible before giving up and
@@ -1271,6 +1299,7 @@ lookup_fastforward(struct namei_state *s
searchdir = *searchdir_ret;
oldnameptr = cnp->cn_nameptr;
oldpathlen = ndp->ni_pathlen;
+ terminal = false;
for (;;) {
foundobj = NULL;
@@ -1304,7 +1333,8 @@ lookup_fastforward(struct namei_state *s
/*
* Can't deal with last component when modifying; this needs
* searchdir locked and VOP_LOOKUP() called (which can and
- * does modify state, despite the name).
+ * does modify state, despite the name). NB: this case means
+ * terminal is never set true when LOCKPARENT.
*/
if ((cnp->cn_flags & ISLASTCN) != 0) {
if (cnp->cn_nameiop != LOOKUP ||
@@ -1338,26 +1368,61 @@ lookup_fastforward(struct namei_state *s
error = EOPNOTSUPP;
} else {
error = ENOENT;
+ terminal = ((cnp->cn_flags & ISLASTCN) != 0);
}
break;
}
/*
- * Stop and get a hold on the vnode if there's something
- * that can't be handled here:
- *
- * - we've reached the last component.
- * - or encountered a mount point that needs to be crossed.
- * - or encountered something other than a directory.
- */
- if ((cnp->cn_flags & ISLASTCN) != 0 ||
- foundobj->v_type != VDIR ||
- (foundobj->v_type == VDIR &&
- foundobj->v_mountedhere != NULL)) {
+ * Stop and get a hold on the vnode if we've encountered
+ * something other than a dirctory.
+ */
+ if (foundobj->v_type != VDIR) {
+ error = vcache_tryvget(foundobj);
+ if (error != 0) {
+ foundobj = NULL;
+ error = EOPNOTSUPP;
+ }
+ break;
+ }
+
+ /*
+ * Try to cross mountpoints, bearing in mind that they can
+ * be stacked. If at any point we can't go further, stop
+ * and try to get a reference on the vnode. If we are able
+ * to get a ref then lookup_crossmount() will take care of
+ * it, otherwise we'll fall through to lookup_once().
+ */
+ if (foundobj->v_mountedhere != NULL) {
+ while (foundobj->v_mountedhere != NULL &&
+ (cnp->cn_flags & NOCROSSMOUNT) == 0 &&
+ cache_cross_mount(&foundobj, &plock)) {
+ KASSERT(foundobj != NULL);
+ KASSERT(foundobj->v_type == VDIR);
+ }
+ if (foundobj->v_mountedhere != NULL) {
+ error = vcache_tryvget(foundobj);
+ if (error != 0) {
+ foundobj = NULL;
+ error = EOPNOTSUPP;
+ }
+ break;
+ } else {
+ searchdir = NULL;
+ }
+ }
+
+ /*
+ * Time to stop if we found the last component & traversed
+ * all mounts.
+ */
+ if ((cnp->cn_flags & ISLASTCN) != 0) {
error = vcache_tryvget(foundobj);
if (error != 0) {
foundobj = NULL;
error = EOPNOTSUPP;
+ } else {
+ terminal = (foundobj->v_type != VLNK);
}
break;
}
@@ -1371,14 +1436,28 @@ lookup_fastforward(struct namei_state *s
searchdir = foundobj;
}
- /*
- * If we ended up with a new search dir, ref it before dropping the
- * namecache's lock. The lock prevents both searchdir and foundobj
- * from disappearing. If we can't ref the new searchdir, we have a
- * bit of a problem. Roll back the fastforward to the beginning and
- * let lookup_once() take care of it.
- */
- if (searchdir != *searchdir_ret) {
+ if (terminal) {
+ /*
+ * If we exited the loop above having successfully located
+ * the last component with a zero error code, and it's not a
+ * symbolic link, then the parent directory is not needed.
+ * Release reference to the starting parent and make the
+ * terminal parent disappear into thin air.
+ */
+ KASSERT(plock != NULL);
+ rw_exit(plock);
+ vrele(*searchdir_ret);
+ *searchdir_ret = NULL;
+ } else if (searchdir != *searchdir_ret) {
+ /*
+ * Otherwise we need to return the parent. If we ended up
+ * with a new search dir, ref it before dropping the
+ * namecache's lock. The lock prevents both searchdir and
+ * foundobj from disappearing. If we can't ref the new
+ * searchdir, we have a bit of a problem. Roll back the
+ * fastforward to the beginning and let lookup_once() take
+ * care of it.
+ */
error2 = vcache_tryvget(searchdir);
KASSERT(plock != NULL);
rw_exit(plock);
Index: src/sys/sys/namei.src
diff -u src/sys/sys/namei.src:1.57 src/sys/sys/namei.src:1.58
--- src/sys/sys/namei.src:1.57 Wed May 27 02:03:30 2020
+++ src/sys/sys/namei.src Sat May 30 20:16:14 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: namei.src,v 1.57 2020/05/27 02:03:30 rin Exp $ */
+/* $NetBSD: namei.src,v 1.58 2020/05/30 20:16:14 ad Exp $ */
/*
* Copyright (c) 1985, 1989, 1991, 1993
@@ -300,6 +300,9 @@ bool cache_have_id(struct vnode *);
void cache_vnode_init(struct vnode * );
void cache_vnode_fini(struct vnode * );
void cache_cpu_init(struct cpu_info *);
+void cache_enter_mount(struct vnode *, struct vnode *);
+bool cache_cross_mount(struct vnode **, krwlock_t **);
+bool cache_lookup_mount(struct vnode *, struct vnode **);
void nchinit(void);
void namecache_count_pass2(void);
